2 * Copyright (C) 2011-2012 ARM Limited. All rights reserved.
4 * This program is free software and is provided to you under the terms of the GNU General Public License version 2
5 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
7 * A copy of the licence is included with the program, and can also be obtained from Free Software
8 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
11 #include "mali_gp_scheduler.h"
12 #include "mali_kernel_common.h"
14 #include "mali_osk_list.h"
15 #include "mali_scheduler.h"
17 #include "mali_gp_job.h"
18 #include "mali_group.h"
21 enum mali_gp_slot_state
23 MALI_GP_SLOT_STATE_IDLE,
24 MALI_GP_SLOT_STATE_WORKING,
27 /* A render slot is an entity which jobs can be scheduled onto */
30 struct mali_group *group;
32 * We keep track of the state here as well as in the group object
33 * so we don't need to take the group lock so often (and also avoid clutter with the working lock)
35 enum mali_gp_slot_state state;
39 static u32 gp_version = 0;
40 static _MALI_OSK_LIST_HEAD(job_queue); /* List of jobs with some unscheduled work */
41 static struct mali_gp_slot slot;
43 /* Variables to allow safe pausing of the scheduler */
44 static _mali_osk_wait_queue_t *gp_scheduler_working_wait_queue = NULL;
45 static u32 pause_count = 0;
47 static mali_bool mali_gp_scheduler_is_suspended(void);
49 static _mali_osk_lock_t *gp_scheduler_lock = NULL;
50 /* Contains tid of thread that locked the scheduler or 0, if not locked */
52 _mali_osk_errcode_t mali_gp_scheduler_initialize(void)
57 _MALI_OSK_INIT_LIST_HEAD(&job_queue);
59 gp_scheduler_lock = _mali_osk_lock_init(_MALI_OSK_LOCKFLAG_ORDERED | _MALI_OSK_LOCKFLAG_SPINLOCK | _MALI_OSK_LOCKFLAG_NONINTERRUPTABLE, 0, _MALI_OSK_LOCK_ORDER_SCHEDULER);
60 if (NULL == gp_scheduler_lock)
62 return _MALI_OSK_ERR_NOMEM;
65 gp_scheduler_working_wait_queue = _mali_osk_wait_queue_init();
66 if (NULL == gp_scheduler_working_wait_queue)
68 _mali_osk_lock_term(gp_scheduler_lock);
69 return _MALI_OSK_ERR_NOMEM;
72 /* Find all the available GP cores */
73 num_groups = mali_group_get_glob_num_groups();
74 for (i = 0; i < num_groups; i++)
76 struct mali_group *group = mali_group_get_glob_group(i);
78 struct mali_gp_core *gp_core = mali_group_get_gp_core(group);
83 /* Retrieve GP version */
84 gp_version = mali_gp_core_get_version(gp_core);
87 slot.state = MALI_GP_SLOT_STATE_IDLE;
88 break; /* There is only one GP, no point in looking for more */
92 return _MALI_OSK_ERR_OK;
95 void mali_gp_scheduler_terminate(void)
97 MALI_DEBUG_ASSERT(MALI_GP_SLOT_STATE_IDLE == slot.state);
98 MALI_DEBUG_ASSERT_POINTER(slot.group);
99 mali_group_delete(slot.group);
101 _mali_osk_wait_queue_term(gp_scheduler_working_wait_queue);
102 _mali_osk_lock_term(gp_scheduler_lock);
105 MALI_STATIC_INLINE void mali_gp_scheduler_lock(void)
107 if(_MALI_OSK_ERR_OK != _mali_osk_lock_wait(gp_scheduler_lock, _MALI_OSK_LOCKMODE_RW))
109 /* Non-interruptable lock failed: this should never happen. */
110 MALI_DEBUG_ASSERT(0);
112 MALI_DEBUG_PRINT(5, ("Mali GP scheduler: GP scheduler lock taken\n"));
115 MALI_STATIC_INLINE void mali_gp_scheduler_unlock(void)
117 MALI_DEBUG_PRINT(5, ("Mali GP scheduler: Releasing GP scheduler lock\n"));
118 _mali_osk_lock_signal(gp_scheduler_lock, _MALI_OSK_LOCKMODE_RW);
122 MALI_STATIC_INLINE void mali_gp_scheduler_assert_locked(void)
124 MALI_DEBUG_ASSERT_LOCK_HELD(gp_scheduler_lock);
126 #define MALI_ASSERT_GP_SCHEDULER_LOCKED() mali_gp_scheduler_assert_locked()
128 #define MALI_ASSERT_GP_SCHEDULER_LOCKED()
131 static void mali_gp_scheduler_schedule(void)
133 struct mali_gp_job *job;
135 mali_gp_scheduler_lock();
137 if (0 < pause_count || MALI_GP_SLOT_STATE_IDLE != slot.state || _mali_osk_list_empty(&job_queue))
139 MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Nothing to schedule (paused=%u, idle slots=%u)\n",
140 pause_count, MALI_GP_SLOT_STATE_IDLE == slot.state ? 1 : 0));
141 mali_gp_scheduler_unlock();
142 return; /* Nothing to do, so early out */
145 /* Get (and remove) next job in queue */
146 job = _MALI_OSK_LIST_ENTRY(job_queue.next, struct mali_gp_job, list);
147 _mali_osk_list_del(&job->list);
149 /* Mark slot as busy */
150 slot.state = MALI_GP_SLOT_STATE_WORKING;
152 mali_gp_scheduler_unlock();
154 MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Starting job %u (0x%08X)\n", mali_gp_job_get_id(job), job));
156 mali_group_lock(slot.group);
158 if (_MALI_OSK_ERR_OK != mali_group_start_gp_job(slot.group, job))
160 MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Failed to start GP job\n"));
161 MALI_DEBUG_ASSERT(0); /* @@@@ todo: this cant fail on Mali-300+, no need to implement put back of job */
164 mali_group_unlock(slot.group);
167 /* @@@@ todo: pass the job in as a param to this function, so that we don't have to take the scheduler lock again */
168 static void mali_gp_scheduler_schedule_on_group(struct mali_group *group)
170 struct mali_gp_job *job;
172 MALI_DEBUG_ASSERT_LOCK_HELD(group->lock);
173 MALI_DEBUG_ASSERT_LOCK_HELD(gp_scheduler_lock);
175 if (0 < pause_count || MALI_GP_SLOT_STATE_IDLE != slot.state || _mali_osk_list_empty(&job_queue))
177 mali_gp_scheduler_unlock();
178 MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Nothing to schedule (paused=%u, idle slots=%u)\n",
179 pause_count, MALI_GP_SLOT_STATE_IDLE == slot.state ? 1 : 0));
180 return; /* Nothing to do, so early out */
183 /* Get (and remove) next job in queue */
184 job = _MALI_OSK_LIST_ENTRY(job_queue.next, struct mali_gp_job, list);
185 _mali_osk_list_del(&job->list);
187 /* Mark slot as busy */
188 slot.state = MALI_GP_SLOT_STATE_WORKING;
190 mali_gp_scheduler_unlock();
192 MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Starting job %u (0x%08X)\n", mali_gp_job_get_id(job), job));
194 if (_MALI_OSK_ERR_OK != mali_group_start_gp_job(slot.group, job))
196 MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Failed to start GP job\n"));
197 MALI_DEBUG_ASSERT(0); /* @@@@ todo: this cant fail on Mali-300+, no need to implement put back of job */
201 static void mali_gp_scheduler_return_job_to_user(struct mali_gp_job *job, mali_bool success)
203 _mali_uk_gp_job_finished_s *jobres = job->finished_notification->result_buffer;
204 _mali_osk_memset(jobres, 0, sizeof(_mali_uk_gp_job_finished_s)); /* @@@@ can be removed once we initialize all members in this struct */
205 jobres->user_job_ptr = mali_gp_job_get_user_id(job);
206 if (MALI_TRUE == success)
208 jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS;
212 jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR;
215 jobres->heap_current_addr = mali_gp_job_get_current_heap_addr(job);
216 jobres->perf_counter0 = mali_gp_job_get_perf_counter_value0(job);
217 jobres->perf_counter1 = mali_gp_job_get_perf_counter_value1(job);
219 mali_session_send_notification(mali_gp_job_get_session(job), job->finished_notification);
220 job->finished_notification = NULL;
222 mali_gp_job_delete(job);
225 void mali_gp_scheduler_job_done(struct mali_group *group, struct mali_gp_job *job, mali_bool success)
227 MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Job %u (0x%08X) completed (%s)\n", mali_gp_job_get_id(job), job, success ? "success" : "failure"));
229 mali_gp_scheduler_return_job_to_user(job, success);
231 mali_gp_scheduler_lock();
233 /* Mark slot as idle again */
234 slot.state = MALI_GP_SLOT_STATE_IDLE;
236 /* If paused, then this was the last job, so wake up sleeping workers */
239 _mali_osk_wait_queue_wake_up(gp_scheduler_working_wait_queue);
242 mali_gp_scheduler_schedule_on_group(group);
244 /* It is ok to do this after schedule, since START/STOP is simply ++ and -- anyways */
245 mali_pm_core_event(MALI_CORE_EVENT_GP_STOP);
248 void mali_gp_scheduler_oom(struct mali_group *group, struct mali_gp_job *job)
250 _mali_uk_gp_job_suspended_s * jobres;
251 _mali_osk_notification_t * notification;
253 mali_gp_scheduler_lock();
255 notification = job->oom_notification;
256 job->oom_notification = NULL;
257 slot.returned_cookie = mali_gp_job_get_id(job);
259 jobres = (_mali_uk_gp_job_suspended_s *)notification->result_buffer;
260 jobres->user_job_ptr = mali_gp_job_get_user_id(job);
261 jobres->cookie = mali_gp_job_get_id(job);
263 mali_gp_scheduler_unlock();
265 jobres->reason = _MALIGP_JOB_SUSPENDED_OUT_OF_MEMORY;
267 mali_session_send_notification(mali_gp_job_get_session(job), notification);
270 * If this function failed, then we could return the job to user space right away,
271 * but there is a job timer anyway that will do that eventually.
272 * This is not exactly a common case anyway.
276 void mali_gp_scheduler_suspend(void)
278 mali_gp_scheduler_lock();
279 pause_count++; /* Increment the pause_count so that no more jobs will be scheduled */
280 mali_gp_scheduler_unlock();
282 _mali_osk_wait_queue_wait_event(gp_scheduler_working_wait_queue, mali_gp_scheduler_is_suspended);
285 void mali_gp_scheduler_resume(void)
287 mali_gp_scheduler_lock();
288 pause_count--; /* Decrement pause_count to allow scheduling again (if it reaches 0) */
289 mali_gp_scheduler_unlock();
290 if (0 == pause_count)
292 mali_gp_scheduler_schedule();
296 _mali_osk_errcode_t _mali_ukk_gp_start_job(void *ctx, _mali_uk_gp_start_job_s *uargs)
298 struct mali_session_data *session;
299 struct mali_gp_job *job;
301 MALI_DEBUG_ASSERT_POINTER(uargs);
302 MALI_DEBUG_ASSERT_POINTER(ctx);
304 session = (struct mali_session_data*)ctx;
306 job = mali_gp_job_create(session, uargs, mali_scheduler_get_new_id());
309 return _MALI_OSK_ERR_NOMEM;
312 #if PROFILING_SKIP_PP_AND_GP_JOBS
313 #warning GP jobs will not be executed
314 mali_gp_scheduler_return_job_to_user(job, MALI_TRUE);
315 return _MALI_OSK_ERR_OK;
318 mali_pm_core_event(MALI_CORE_EVENT_GP_START);
320 mali_gp_scheduler_lock();
321 _mali_osk_list_addtail(&job->list, &job_queue);
322 mali_gp_scheduler_unlock();
324 MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Job %u (0x%08X) queued\n", mali_gp_job_get_id(job), job));
326 mali_gp_scheduler_schedule();
328 return _MALI_OSK_ERR_OK;
331 _mali_osk_errcode_t _mali_ukk_get_gp_number_of_cores(_mali_uk_get_gp_number_of_cores_s *args)
333 MALI_DEBUG_ASSERT_POINTER(args);
334 MALI_CHECK_NON_NULL(args->ctx, _MALI_OSK_ERR_INVALID_ARGS);
335 args->number_of_cores = 1;
336 return _MALI_OSK_ERR_OK;
339 _mali_osk_errcode_t _mali_ukk_get_gp_core_version(_mali_uk_get_gp_core_version_s *args)
341 MALI_DEBUG_ASSERT_POINTER(args);
342 MALI_CHECK_NON_NULL(args->ctx, _MALI_OSK_ERR_INVALID_ARGS);
343 args->version = gp_version;
344 return _MALI_OSK_ERR_OK;
347 _mali_osk_errcode_t _mali_ukk_gp_suspend_response(_mali_uk_gp_suspend_response_s *args)
349 struct mali_session_data *session;
350 struct mali_gp_job *resumed_job;
351 _mali_osk_notification_t *new_notification = 0;
353 MALI_DEBUG_ASSERT_POINTER(args);
355 if (NULL == args->ctx)
357 return _MALI_OSK_ERR_INVALID_ARGS;
360 session = (struct mali_session_data*)args->ctx;
363 return _MALI_OSK_ERR_FAULT;
366 if (_MALIGP_JOB_RESUME_WITH_NEW_HEAP == args->code)
368 new_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_STALLED, sizeof(_mali_uk_gp_job_suspended_s));
370 if (NULL == new_notification)
372 MALI_PRINT_ERROR(("Mali GP scheduler: Failed to allocate notification object. Will abort GP job.\n"));
373 mali_group_lock(slot.group);
374 mali_group_abort_gp_job(slot.group, args->cookie);
375 mali_group_unlock(slot.group);
376 return _MALI_OSK_ERR_FAULT;
380 mali_group_lock(slot.group);
382 if (_MALIGP_JOB_RESUME_WITH_NEW_HEAP == args->code)
384 MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Resuming job %u with new heap; 0x%08X - 0x%08X\n", args->cookie, args->arguments[0], args->arguments[1]));
386 resumed_job = mali_group_resume_gp_with_new_heap(slot.group, args->cookie, args->arguments[0], args->arguments[1]);
387 if (NULL != resumed_job)
389 /* @@@@ todo: move this and other notification handling into the job object itself */
390 resumed_job->oom_notification = new_notification;
391 mali_group_unlock(slot.group);
392 return _MALI_OSK_ERR_OK;
396 mali_group_unlock(slot.group);
397 _mali_osk_notification_delete(new_notification);
398 return _MALI_OSK_ERR_FAULT;
402 MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Aborting job %u, no new heap provided\n", args->cookie));
403 mali_group_abort_gp_job(slot.group, args->cookie);
404 mali_group_unlock(slot.group);
405 return _MALI_OSK_ERR_OK;
408 void mali_gp_scheduler_abort_session(struct mali_session_data *session)
410 struct mali_gp_job *job, *tmp;
412 mali_gp_scheduler_lock();
413 MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Aborting all jobs from session 0x%08x\n", session));
415 /* Check queue for jobs and remove */
416 _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &job_queue, struct mali_gp_job, list)
418 if (mali_gp_job_get_session(job) == session)
420 MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Removing GP job 0x%08x from queue\n", job));
421 _mali_osk_list_del(&(job->list));
422 mali_gp_job_delete(job);
424 mali_pm_core_event(MALI_CORE_EVENT_GP_STOP);
428 mali_gp_scheduler_unlock();
430 mali_group_abort_session(slot.group, session);
433 static mali_bool mali_gp_scheduler_is_suspended(void)
437 mali_gp_scheduler_lock();
438 ret = pause_count > 0 && slot.state == MALI_GP_SLOT_STATE_IDLE;
439 mali_gp_scheduler_unlock();
445 #if MALI_STATE_TRACKING
446 u32 mali_gp_scheduler_dump_state(char *buf, u32 size)
450 n += _mali_osk_snprintf(buf + n, size - n, "GP\n");
451 n += _mali_osk_snprintf(buf + n, size - n, "\tQueue is %s\n", _mali_osk_list_empty(&job_queue) ? "empty" : "not empty");
453 n += mali_group_dump_state(slot.group, buf + n, size - n);
454 n += _mali_osk_snprintf(buf + n, size - n, "\n");
460 void mali_gp_scheduler_reset_all_groups(void)
462 if (NULL != slot.group)
464 mali_group_reset(slot.group);
468 void mali_gp_scheduler_zap_all_active(struct mali_session_data *session)
470 if (NULL != slot.group)
472 mali_group_zap_session(slot.group, session);