3 * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
21 * @file mali_kbase_js.h
28 #include "mali_kbase_js_defs.h"
29 #include "mali_kbase_js_policy.h"
30 #include "mali_kbase_defs.h"
31 #include "mali_kbase_debug.h"
33 #include "mali_kbase_js_ctx_attr.h"
36 * @addtogroup base_api
41 * @addtogroup base_kbase_api
46 * @addtogroup kbase_js Job Scheduler Internal APIs
49 * These APIs are Internal to KBase and are available for use by the
50 * @ref kbase_js_policy "Job Scheduler Policy APIs"
54 * @brief Initialize the Job Scheduler
56 * The struct kbasep_js_device_data sub-structure of \a kbdev must be zero
57 * initialized before passing to the kbasep_js_devdata_init() function. This is
58 * to give efficient error path code.
60 int kbasep_js_devdata_init(struct kbase_device * const kbdev);
63 * @brief Halt the Job Scheduler.
65 * It is safe to call this on \a kbdev even if it the kbasep_js_device_data
66 * sub-structure was never initialized/failed initialization, to give efficient
69 * For this to work, the struct kbasep_js_device_data sub-structure of \a kbdev must
70 * be zero initialized before passing to the kbasep_js_devdata_init()
71 * function. This is to give efficient error path code.
73 * It is a Programming Error to call this whilst there are still kbase_context
74 * structures registered with this scheduler.
77 void kbasep_js_devdata_halt(struct kbase_device *kbdev);
80 * @brief Terminate the Job Scheduler
82 * It is safe to call this on \a kbdev even if it the kbasep_js_device_data
83 * sub-structure was never initialized/failed initialization, to give efficient
86 * For this to work, the struct kbasep_js_device_data sub-structure of \a kbdev must
87 * be zero initialized before passing to the kbasep_js_devdata_init()
88 * function. This is to give efficient error path code.
90 * It is a Programming Error to call this whilst there are still kbase_context
91 * structures registered with this scheduler.
93 void kbasep_js_devdata_term(struct kbase_device *kbdev);
96 * @brief Initialize the Scheduling Component of a struct kbase_context on the Job Scheduler.
98 * This effectively registers a struct kbase_context with a Job Scheduler.
100 * It does not register any jobs owned by the struct kbase_context with the scheduler.
101 * Those must be separately registered by kbasep_js_add_job().
103 * The struct kbase_context must be zero intitialized before passing to the
104 * kbase_js_init() function. This is to give efficient error path code.
106 int kbasep_js_kctx_init(struct kbase_context * const kctx);
109 * @brief Terminate the Scheduling Component of a struct kbase_context on the Job Scheduler
111 * This effectively de-registers a struct kbase_context from its Job Scheduler
113 * It is safe to call this on a struct kbase_context that has never had or failed
114 * initialization of its jctx.sched_info member, to give efficient error-path
117 * For this to work, the struct kbase_context must be zero intitialized before passing
118 * to the kbase_js_init() function.
120 * It is a Programming Error to call this whilst there are still jobs
121 * registered with this context.
123 void kbasep_js_kctx_term(struct kbase_context *kctx);
126 * @brief Add a job chain to the Job Scheduler, and take necessary actions to
127 * schedule the context/run the job.
129 * This atomically does the following:
130 * - Update the numbers of jobs information
131 * - Add the job to the run pool if necessary (part of init_job)
133 * Once this is done, then an appropriate action is taken:
134 * - If the ctx is scheduled, it attempts to start the next job (which might be
136 * - Otherwise, and if this is the first job on the context, it enqueues it on
139 * The Policy's Queue can be updated by this in the following ways:
140 * - In the above case that this is the first job on the context
141 * - If the context is high priority and the context is not scheduled, then it
142 * could cause the Policy to schedule out a low-priority context, allowing
143 * this context to be scheduled in.
145 * If the context is already scheduled on the RunPool, then adding a job to it
146 * is guarenteed not to update the Policy Queue. And so, the caller is
147 * guarenteed to not need to try scheduling a context from the Run Pool - it
148 * can safely assert that the result is false.
150 * It is a programming error to have more than U32_MAX jobs in flight at a time.
152 * The following locking conditions are made on the caller:
153 * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex.
154 * - it must \em not hold kbasep_js_device_data::runpool_irq::lock (as this will be
155 * obtained internally)
156 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
157 * obtained internally)
158 * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally).
160 * @return true indicates that the Policy Queue was updated, and so the
161 * caller will need to try scheduling a context onto the Run Pool.
162 * @return false indicates that no updates were made to the Policy Queue,
163 * so no further action is required from the caller. This is \b always returned
164 * when the context is currently scheduled.
166 bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom);
169 * @brief Remove a job chain from the Job Scheduler, except for its 'retained state'.
171 * Completely removing a job requires several calls:
172 * - kbasep_js_copy_atom_retained_state(), to capture the 'retained state' of
174 * - kbasep_js_remove_job(), to partially remove the atom from the Job Scheduler
175 * - kbasep_js_runpool_release_ctx_and_katom_retained_state(), to release the
176 * remaining state held as part of the job having been run.
178 * In the common case of atoms completing normally, this set of actions is more optimal for spinlock purposes than having kbasep_js_remove_job() handle all of the actions.
180 * In the case of cancelling atoms, it is easier to call kbasep_js_remove_cancelled_job(), which handles all the necessary actions.
182 * It is a programming error to call this when:
183 * - \a atom is not a job belonging to kctx.
184 * - \a atom has already been removed from the Job Scheduler.
185 * - \a atom is still in the runpool:
186 * - it has not been removed with kbasep_js_policy_dequeue_job()
187 * - or, it has not been removed with kbasep_js_policy_dequeue_job_irq()
189 * Do not use this for removing jobs being killed by kbase_jd_cancel() - use
190 * kbasep_js_remove_cancelled_job() instead.
192 * The following locking conditions are made on the caller:
193 * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex.
196 void kbasep_js_remove_job(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *atom);
199 * @brief Completely remove a job chain from the Job Scheduler, in the case
200 * where the job chain was cancelled.
202 * This is a variant of kbasep_js_remove_job() that takes care of removing all
203 * of the retained state too. This is generally useful for cancelled atoms,
204 * which need not be handled in an optimal way.
206 * It is a programming error to call this when:
207 * - \a atom is not a job belonging to kctx.
208 * - \a atom has already been removed from the Job Scheduler.
209 * - \a atom is still in the runpool:
210 * - it is not being killed with kbasep_jd_cancel()
211 * - or, it has not been removed with kbasep_js_policy_dequeue_job()
212 * - or, it has not been removed with kbasep_js_policy_dequeue_job_irq()
214 * The following locking conditions are made on the caller:
215 * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex.
216 * - it must \em not hold the kbasep_js_device_data::runpool_irq::lock, (as this will be
217 * obtained internally)
218 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this could be
219 * obtained internally)
221 * @return true indicates that ctx attributes have changed and the caller
222 * should call kbase_js_sched_all() to try to run more jobs
223 * @return false otherwise
225 bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev,
226 struct kbase_context *kctx,
227 struct kbase_jd_atom *katom);
230 * @brief Refcount a context as being busy, preventing it from being scheduled
233 * @note This function can safely be called from IRQ context.
235 * The following locking conditions are made on the caller:
236 * - it must \em not hold the kbasep_js_device_data::runpool_irq::lock, because
237 * it will be used internally.
239 * @return value != false if the retain succeeded, and the context will not be scheduled out.
240 * @return false if the retain failed (because the context is being/has been scheduled out).
242 bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx);
245 * @brief Refcount a context as being busy, preventing it from being scheduled
248 * @note This function can safely be called from IRQ context.
250 * The following locks must be held by the caller:
251 * - kbasep_js_device_data::runpool_irq::lock
253 * @return value != false if the retain succeeded, and the context will not be scheduled out.
254 * @return false if the retain failed (because the context is being/has been scheduled out).
256 bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, struct kbase_context *kctx);
259 * @brief Lookup a context in the Run Pool based upon its current address space
260 * and ensure that is stays scheduled in.
262 * The context is refcounted as being busy to prevent it from scheduling
263 * out. It must be released with kbasep_js_runpool_release_ctx() when it is no
264 * longer required to stay scheduled in.
266 * @note This function can safely be called from IRQ context.
268 * The following locking conditions are made on the caller:
269 * - it must \em not hold the kbasep_js_device_data::runpoool_irq::lock, because
270 * it will be used internally. If the runpool_irq::lock is already held, then
271 * the caller should use kbasep_js_runpool_lookup_ctx_nolock() instead.
273 * @return a valid struct kbase_context on success, which has been refcounted as being busy.
274 * @return NULL on failure, indicating that no context was found in \a as_nr
276 struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, int as_nr);
279 * kbasep_js_runpool_lookup_ctx_nolock - Lookup a context in the Run Pool based
280 * upon its current address space and ensure that is stays scheduled in.
281 * @kbdev: Device pointer
282 * @as_nr: Address space to lookup
284 * The context is refcounted as being busy to prevent it from scheduling
285 * out. It must be released with kbasep_js_runpool_release_ctx() when it is no
286 * longer required to stay scheduled in.
288 * Note: This function can safely be called from IRQ context.
290 * The following locking conditions are made on the caller:
291 * - it must the kbasep_js_device_data::runpoool_irq::lock.
293 * Return: a valid struct kbase_context on success, which has been refcounted as
295 * NULL on failure, indicating that no context was found in \a as_nr
297 struct kbase_context *kbasep_js_runpool_lookup_ctx_nolock(
298 struct kbase_device *kbdev, int as_nr);
301 * @brief Handling the requeuing/killing of a context that was evicted from the
302 * policy queue or runpool.
304 * This should be used whenever handing off a context that has been evicted
305 * from the policy queue or the runpool:
306 * - If the context is not dying and has jobs, it gets re-added to the policy
308 * - Otherwise, it is not added
310 * In addition, if the context is dying the jobs are killed asynchronously.
312 * In all cases, the Power Manager active reference is released
313 * (kbase_pm_context_idle()) whenever the has_pm_ref parameter is true. \a
314 * has_pm_ref must be set to false whenever the context was not previously in
315 * the runpool and does not hold a Power Manager active refcount. Note that
316 * contexts in a rollback of kbasep_js_try_schedule_head_ctx() might have an
317 * active refcount even though they weren't in the runpool.
319 * The following locking conditions are made on the caller:
320 * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex.
321 * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be
322 * obtained internally)
324 void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, bool has_pm_ref);
327 * @brief Release a refcount of a context being busy, allowing it to be
330 * When the refcount reaches zero and the context \em might be scheduled out
331 * (depending on whether the Scheudling Policy has deemed it so, or if it has run
334 * If the context does get scheduled out, then The following actions will be
335 * taken as part of deschduling a context:
336 * - For the context being descheduled:
337 * - If the context is in the processing of dying (all the jobs are being
338 * removed from it), then descheduling also kills off any jobs remaining in the
340 * - If the context is not dying, and any jobs remain after descheduling the
341 * context then it is re-enqueued to the Policy's Queue.
342 * - Otherwise, the context is still known to the scheduler, but remains absent
343 * from the Policy Queue until a job is next added to it.
344 * - In all descheduling cases, the Power Manager active reference (obtained
345 * during kbasep_js_try_schedule_head_ctx()) is released (kbase_pm_context_idle()).
347 * Whilst the context is being descheduled, this also handles actions that
348 * cause more atoms to be run:
349 * - Attempt submitting atoms when the Context Attributes on the Runpool have
350 * changed. This is because the context being scheduled out could mean that
351 * there are more opportunities to run atoms.
352 * - Attempt submitting to a slot that was previously blocked due to affinity
353 * restrictions. This is usually only necessary when releasing a context
354 * happens as part of completing a previous job, but is harmless nonetheless.
355 * - Attempt scheduling in a new context (if one is available), and if necessary,
356 * running a job from that new context.
358 * Unlike retaining a context in the runpool, this function \b cannot be called
361 * It is a programming error to call this on a \a kctx that is not currently
362 * scheduled, or that already has a zero refcount.
364 * The following locking conditions are made on the caller:
365 * - it must \em not hold the kbasep_js_device_data::runpool_irq::lock, because
366 * it will be used internally.
367 * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex.
368 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
369 * obtained internally)
370 * - it must \em not hold the kbase_device::as[n].transaction_mutex (as this will be obtained internally)
371 * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be
372 * obtained internally)
375 void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx);
378 * @brief Variant of kbasep_js_runpool_release_ctx() that handles additional
379 * actions from completing an atom.
381 * This is usually called as part of completing an atom and releasing the
382 * refcount on the context held by the atom.
384 * Therefore, the extra actions carried out are part of handling actions queued
385 * on a completed atom, namely:
386 * - Releasing the atom's context attributes
387 * - Retrying the submission on a particular slot, because we couldn't submit
388 * on that slot from an IRQ handler.
390 * The locking conditions of this function are the same as those for
391 * kbasep_js_runpool_release_ctx()
393 void kbasep_js_runpool_release_ctx_and_katom_retained_state(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state);
396 * @brief Variant of kbase_js_runpool_release_ctx() that assumes that
397 * kbasep_js_device_data::runpool_mutex and
398 * kbasep_js_kctx_info::ctx::jsctx_mutex are held by the caller, and does not
399 * attempt to schedule new contexts.
401 void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev,
402 struct kbase_context *kctx);
405 * @brief Schedule in a privileged context
407 * This schedules a context in regardless of the context priority.
408 * If the runpool is full, a context will be forced out of the runpool and the function will wait
409 * for the new context to be scheduled in.
410 * The context will be kept scheduled in (and the corresponding address space reserved) until
411 * kbasep_js_release_privileged_ctx is called).
413 * The following locking conditions are made on the caller:
414 * - it must \em not hold the kbasep_js_device_data::runpool_irq::lock, because
415 * it will be used internally.
416 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
417 * obtained internally)
418 * - it must \em not hold the kbase_device::as[n].transaction_mutex (as this will be obtained internally)
419 * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally).
420 * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex, because it will
421 * be used internally.
424 void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx);
427 * @brief Release a privileged context, allowing it to be scheduled out.
429 * See kbasep_js_runpool_release_ctx for potential side effects.
431 * The following locking conditions are made on the caller:
432 * - it must \em not hold the kbasep_js_device_data::runpool_irq::lock, because
433 * it will be used internally.
434 * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex.
435 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
436 * obtained internally)
437 * - it must \em not hold the kbase_device::as[n].transaction_mutex (as this will be obtained internally)
440 void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx);
443 * @brief Try to submit the next job on each slot
445 * The following locks may be used:
446 * - kbasep_js_device_data::runpool_mutex
447 * - kbasep_js_device_data::runpool_irq::lock
449 void kbase_js_try_run_jobs(struct kbase_device *kbdev);
452 * @brief Suspend the job scheduler during a Power Management Suspend event.
454 * Causes all contexts to be removed from the runpool, and prevents any
455 * contexts from (re)entering the runpool.
457 * This does not handle suspending the one privileged context: the caller must
458 * instead do this by by suspending the GPU HW Counter Instrumentation.
460 * This will eventually cause all Power Management active references held by
461 * contexts on the runpool to be released, without running any more atoms.
463 * The caller must then wait for all Power Mangement active refcount to become
464 * zero before completing the suspend.
466 * The emptying mechanism may take some time to complete, since it can wait for
467 * jobs to complete naturally instead of forcing them to end quickly. However,
468 * this is bounded by the Job Scheduling Policy's Job Timeouts. Hence, this
469 * function is guaranteed to complete in a finite time whenever the Job
470 * Scheduling Policy implements Job Timeouts (such as those done by CFS).
472 void kbasep_js_suspend(struct kbase_device *kbdev);
475 * @brief Resume the Job Scheduler after a Power Management Resume event.
477 * This restores the actions from kbasep_js_suspend():
478 * - Schedules contexts back into the runpool
479 * - Resumes running atoms on the GPU
481 void kbasep_js_resume(struct kbase_device *kbdev);
484 * @brief Submit an atom to the job scheduler.
486 * The atom is enqueued on the context's ringbuffer. The caller must have
487 * ensured that all dependencies can be represented in the ringbuffer.
489 * Caller must hold jctx->lock
491 * @param[in] kctx Context pointer
492 * @param[in] atom Pointer to the atom to submit
494 * @return Whether the context requires to be enqueued. */
495 bool kbase_js_dep_resolved_submit(struct kbase_context *kctx,
496 struct kbase_jd_atom *katom);
499 * jsctx_ll_flush_to_rb() - Pushes atoms from the linked list to ringbuffer.
500 * @kctx: Context Pointer
501 * @prio: Priority (specifies the queue together with js).
502 * @js: Job slot (specifies the queue together with prio).
504 * Pushes all possible atoms from the linked list to the ringbuffer.
505 * Number of atoms are limited to free space in the ringbuffer and
506 * number of available atoms in the linked list.
509 void jsctx_ll_flush_to_rb(struct kbase_context *kctx, int prio, int js);
511 * @brief Pull an atom from a context in the job scheduler for execution.
513 * The atom will not be removed from the ringbuffer at this stage.
515 * The HW access lock must be held when calling this function.
517 * @param[in] kctx Context to pull from
518 * @param[in] js Job slot to pull from
519 * @return Pointer to an atom, or NULL if there are no atoms for this
520 * slot that can be currently run.
522 struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js);
525 * @brief Return an atom to the job scheduler ringbuffer.
527 * An atom is 'unpulled' if execution is stopped but intended to be returned to
528 * later. The most common reason for this is that the atom has been
531 * Note that if multiple atoms are to be 'unpulled', they must be returned in
532 * the reverse order to which they were originally pulled. It is a programming
533 * error to return atoms in any other order.
535 * The HW access lock must be held when calling this function.
537 * @param[in] kctx Context pointer
538 * @param[in] atom Pointer to the atom to unpull
540 void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom);
543 * @brief Complete an atom from jd_done_worker(), removing it from the job
544 * scheduler ringbuffer.
546 * If the atom failed then all dependee atoms marked for failure propagation
549 * @param[in] kctx Context pointer
550 * @param[in] katom Pointer to the atom to complete
551 * @return true if the context is now idle (no jobs pulled)
554 bool kbase_js_complete_atom_wq(struct kbase_context *kctx,
555 struct kbase_jd_atom *katom);
558 * @brief Complete an atom.
560 * Most of the work required to complete an atom will be performed by
563 * The HW access lock must be held when calling this function.
565 * @param[in] katom Pointer to the atom to complete
566 * @param[in] end_timestamp The time that the atom completed (may be NULL)
568 void kbase_js_complete_atom(struct kbase_jd_atom *katom,
569 ktime_t *end_timestamp);
572 * @brief Submit atoms from all available contexts.
574 * This will attempt to submit as many jobs as possible to the provided job
575 * slots. It will exit when either all job slots are full, or all contexts have
578 * @param[in] kbdev Device pointer
579 * @param[in] js_mask Mask of job slots to submit to
581 void kbase_js_sched(struct kbase_device *kbdev, int js_mask);
584 * kbase_jd_zap_context - Attempt to deschedule a context that is being
586 * @kctx: Context pointer
588 * This will attempt to remove a context from any internal job scheduler queues
589 * and perform any other actions to ensure a context will not be submitted
592 * If the context is currently scheduled, then the caller must wait for all
593 * pending jobs to complete before taking any further action.
595 void kbase_js_zap_context(struct kbase_context *kctx);
598 * @brief Validate an atom
600 * This will determine whether the atom can be scheduled onto the GPU. Atoms
601 * with invalid combinations of core requirements will be rejected.
603 * @param[in] kbdev Device pointer
604 * @param[in] katom Atom to validate
605 * @return true if atom is valid
608 bool kbase_js_is_atom_valid(struct kbase_device *kbdev,
609 struct kbase_jd_atom *katom);
616 * @brief Check that a context is allowed to submit jobs on this policy
618 * The purpose of this abstraction is to hide the underlying data size, and wrap up
619 * the long repeated line of code.
621 * As with any bool, never test the return value with true.
623 * The caller must hold kbasep_js_device_data::runpool_irq::lock.
625 static inline bool kbasep_js_is_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx)
629 /* Ensure context really is scheduled in */
630 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
631 KBASE_DEBUG_ASSERT(kctx->jctx.sched_info.ctx.is_scheduled);
633 test_bit = (u16) (1u << kctx->as_nr);
635 return (bool) (js_devdata->runpool_irq.submit_allowed & test_bit);
639 * @brief Allow a context to submit jobs on this policy
641 * The purpose of this abstraction is to hide the underlying data size, and wrap up
642 * the long repeated line of code.
644 * The caller must hold kbasep_js_device_data::runpool_irq::lock.
646 static inline void kbasep_js_set_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx)
650 /* Ensure context really is scheduled in */
651 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
652 KBASE_DEBUG_ASSERT(kctx->jctx.sched_info.ctx.is_scheduled);
654 set_bit = (u16) (1u << kctx->as_nr);
656 dev_dbg(kctx->kbdev->dev, "JS: Setting Submit Allowed on %p (as=%d)", kctx, kctx->as_nr);
658 js_devdata->runpool_irq.submit_allowed |= set_bit;
662 * @brief Prevent a context from submitting more jobs on this policy
664 * The purpose of this abstraction is to hide the underlying data size, and wrap up
665 * the long repeated line of code.
667 * The caller must hold kbasep_js_device_data::runpool_irq::lock.
669 static inline void kbasep_js_clear_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx)
674 /* Ensure context really is scheduled in */
675 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
676 KBASE_DEBUG_ASSERT(kctx->jctx.sched_info.ctx.is_scheduled);
678 clear_bit = (u16) (1u << kctx->as_nr);
679 clear_mask = ~clear_bit;
681 dev_dbg(kctx->kbdev->dev, "JS: Clearing Submit Allowed on %p (as=%d)", kctx, kctx->as_nr);
683 js_devdata->runpool_irq.submit_allowed &= clear_mask;
687 * @brief Manage the 'retry_submit_on_slot' part of a kbase_jd_atom
689 static inline void kbasep_js_clear_job_retry_submit(struct kbase_jd_atom *atom)
691 atom->retry_submit_on_slot = KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID;
695 * Mark a slot as requiring resubmission by carrying that information on a
698 * @note This can ASSERT in debug builds if the submit slot has been set to
699 * something other than the current value for @a js. This is because you might
700 * be unintentionally stopping more jobs being submitted on the old submit
701 * slot, and that might cause a scheduling-hang.
703 * @note If you can guarantee that the atoms for the original slot will be
704 * submitted on some other slot, then call kbasep_js_clear_job_retry_submit()
705 * first to silence the ASSERT.
707 static inline void kbasep_js_set_job_retry_submit_slot(struct kbase_jd_atom *atom, int js)
709 KBASE_DEBUG_ASSERT(0 <= js && js <= BASE_JM_MAX_NR_SLOTS);
710 KBASE_DEBUG_ASSERT((atom->retry_submit_on_slot ==
711 KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID)
712 || (atom->retry_submit_on_slot == js));
714 atom->retry_submit_on_slot = js;
718 * Create an initial 'invalid' atom retained state, that requires no
719 * atom-related work to be done on releasing with
720 * kbasep_js_runpool_release_ctx_and_katom_retained_state()
722 static inline void kbasep_js_atom_retained_state_init_invalid(struct kbasep_js_atom_retained_state *retained_state)
724 retained_state->event_code = BASE_JD_EVENT_NOT_STARTED;
725 retained_state->core_req = KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID;
726 retained_state->retry_submit_on_slot = KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID;
730 * Copy atom state that can be made available after jd_done_nolock() is called
733 static inline void kbasep_js_atom_retained_state_copy(struct kbasep_js_atom_retained_state *retained_state, const struct kbase_jd_atom *katom)
735 retained_state->event_code = katom->event_code;
736 retained_state->core_req = katom->core_req;
737 retained_state->retry_submit_on_slot = katom->retry_submit_on_slot;
738 retained_state->sched_priority = katom->sched_priority;
739 retained_state->device_nr = katom->device_nr;
743 * @brief Determine whether an atom has finished (given its retained state),
744 * and so should be given back to userspace/removed from the system.
746 * Reasons for an atom not finishing include:
747 * - Being soft-stopped (and so, the atom should be resubmitted sometime later)
749 * @param[in] katom_retained_state the retained state of the atom to check
750 * @return false if the atom has not finished
751 * @return !=false if the atom has finished
753 static inline bool kbasep_js_has_atom_finished(const struct kbasep_js_atom_retained_state *katom_retained_state)
755 return (bool) (katom_retained_state->event_code != BASE_JD_EVENT_STOPPED && katom_retained_state->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT);
759 * @brief Determine whether a struct kbasep_js_atom_retained_state is valid
761 * An invalid struct kbasep_js_atom_retained_state is allowed, and indicates that the
762 * code should just ignore it.
764 * @param[in] katom_retained_state the atom's retained state to check
765 * @return false if the retained state is invalid, and can be ignored
766 * @return !=false if the retained state is valid
768 static inline bool kbasep_js_atom_retained_state_is_valid(const struct kbasep_js_atom_retained_state *katom_retained_state)
770 return (bool) (katom_retained_state->core_req != KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID);
773 static inline bool kbasep_js_get_atom_retry_submit_slot(const struct kbasep_js_atom_retained_state *katom_retained_state, int *res)
775 int js = katom_retained_state->retry_submit_on_slot;
778 return (bool) (js >= 0);
781 #if KBASE_DEBUG_DISABLE_ASSERTS == 0
783 * Debug Check the refcount of a context. Only use within ASSERTs
785 * Obtains kbasep_js_device_data::runpool_irq::lock
787 * @return negative value if the context is not scheduled in
788 * @return current refcount of the context if it is scheduled in. The refcount
789 * is not guarenteed to be kept constant.
791 static inline int kbasep_js_debug_check_ctx_refcount(struct kbase_device *kbdev, struct kbase_context *kctx)
794 struct kbasep_js_device_data *js_devdata;
798 KBASE_DEBUG_ASSERT(kbdev != NULL);
799 KBASE_DEBUG_ASSERT(kctx != NULL);
800 js_devdata = &kbdev->js_data;
802 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
804 if (as_nr != KBASEP_AS_NR_INVALID)
805 result = js_devdata->runpool_irq.per_as_data[as_nr].as_busy_refcount;
807 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
811 #endif /* KBASE_DEBUG_DISABLE_ASSERTS == 0 */
814 * @brief Variant of kbasep_js_runpool_lookup_ctx() that can be used when the
815 * context is guarenteed to be already previously retained.
817 * It is a programming error to supply the \a as_nr of a context that has not
818 * been previously retained/has a busy refcount of zero. The only exception is
819 * when there is no ctx in \a as_nr (NULL returned).
821 * The following locking conditions are made on the caller:
822 * - it must \em not hold the kbasep_js_device_data::runpoool_irq::lock, because
823 * it will be used internally.
825 * @return a valid struct kbase_context on success, with a refcount that is guarenteed
826 * to be non-zero and unmodified by this function.
827 * @return NULL on failure, indicating that no context was found in \a as_nr
829 static inline struct kbase_context *kbasep_js_runpool_lookup_ctx_noretain(struct kbase_device *kbdev, int as_nr)
832 struct kbasep_js_device_data *js_devdata;
833 struct kbase_context *found_kctx;
834 struct kbasep_js_per_as_data *js_per_as_data;
836 KBASE_DEBUG_ASSERT(kbdev != NULL);
837 KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS);
838 js_devdata = &kbdev->js_data;
839 js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
841 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
843 found_kctx = js_per_as_data->kctx;
844 KBASE_DEBUG_ASSERT(found_kctx == NULL || js_per_as_data->as_busy_refcount > 0);
846 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
852 * This will provide a conversion from time (us) to ticks of the gpu clock
853 * based on the minimum available gpu frequency.
854 * This is usually good to compute best/worst case (where the use of current
855 * frequency is not valid due to DVFS).
856 * e.g.: when you need the number of cycles to guarantee you won't wait for
857 * longer than 'us' time (you might have a shorter wait).
859 static inline u32 kbasep_js_convert_us_to_gpu_ticks_min_freq(struct kbase_device *kbdev, u32 us)
861 u32 gpu_freq = kbdev->gpu_props.props.core_props.gpu_freq_khz_min;
863 KBASE_DEBUG_ASSERT(0 != gpu_freq);
864 return us * (gpu_freq / 1000);
868 * This will provide a conversion from time (us) to ticks of the gpu clock
869 * based on the maximum available gpu frequency.
870 * This is usually good to compute best/worst case (where the use of current
871 * frequency is not valid due to DVFS).
872 * e.g.: When you need the number of cycles to guarantee you'll wait at least
873 * 'us' amount of time (but you might wait longer).
875 static inline u32 kbasep_js_convert_us_to_gpu_ticks_max_freq(struct kbase_device *kbdev, u32 us)
877 u32 gpu_freq = kbdev->gpu_props.props.core_props.gpu_freq_khz_max;
879 KBASE_DEBUG_ASSERT(0 != gpu_freq);
880 return us * (u32) (gpu_freq / 1000);
884 * This will provide a conversion from ticks of the gpu clock to time (us)
885 * based on the minimum available gpu frequency.
886 * This is usually good to compute best/worst case (where the use of current
887 * frequency is not valid due to DVFS).
888 * e.g.: When you need to know the worst-case wait that 'ticks' cycles will
889 * take (you guarantee that you won't wait any longer than this, but it may
892 static inline u32 kbasep_js_convert_gpu_ticks_to_us_min_freq(struct kbase_device *kbdev, u32 ticks)
894 u32 gpu_freq = kbdev->gpu_props.props.core_props.gpu_freq_khz_min;
896 KBASE_DEBUG_ASSERT(0 != gpu_freq);
897 return ticks / gpu_freq * 1000;
901 * This will provide a conversion from ticks of the gpu clock to time (us)
902 * based on the maximum available gpu frequency.
903 * This is usually good to compute best/worst case (where the use of current
904 * frequency is not valid due to DVFS).
905 * e.g.: When you need to know the best-case wait for 'tick' cycles (you
906 * guarantee to be waiting for at least this long, but it may be longer).
908 static inline u32 kbasep_js_convert_gpu_ticks_to_us_max_freq(struct kbase_device *kbdev, u32 ticks)
910 u32 gpu_freq = kbdev->gpu_props.props.core_props.gpu_freq_khz_max;
912 KBASE_DEBUG_ASSERT(0 != gpu_freq);
913 return ticks / gpu_freq * 1000;
917 * The following locking conditions are made on the caller:
918 * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex.
919 * - The caller must hold the kbasep_js_device_data::runpool_mutex
921 static inline void kbase_js_runpool_inc_context_count(
922 struct kbase_device *kbdev,
923 struct kbase_context *kctx)
925 struct kbasep_js_device_data *js_devdata;
926 struct kbasep_js_kctx_info *js_kctx_info;
928 KBASE_DEBUG_ASSERT(kbdev != NULL);
929 KBASE_DEBUG_ASSERT(kctx != NULL);
931 js_devdata = &kbdev->js_data;
932 js_kctx_info = &kctx->jctx.sched_info;
934 lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
935 lockdep_assert_held(&js_devdata->runpool_mutex);
937 /* Track total contexts */
938 KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running < S8_MAX);
939 ++(js_devdata->nr_all_contexts_running);
941 if ((js_kctx_info->ctx.flags & KBASE_CTX_FLAG_SUBMIT_DISABLED) == 0) {
942 /* Track contexts that can submit jobs */
943 KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running <
945 ++(js_devdata->nr_user_contexts_running);
950 * The following locking conditions are made on the caller:
951 * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex.
952 * - The caller must hold the kbasep_js_device_data::runpool_mutex
954 static inline void kbase_js_runpool_dec_context_count(
955 struct kbase_device *kbdev,
956 struct kbase_context *kctx)
958 struct kbasep_js_device_data *js_devdata;
959 struct kbasep_js_kctx_info *js_kctx_info;
961 KBASE_DEBUG_ASSERT(kbdev != NULL);
962 KBASE_DEBUG_ASSERT(kctx != NULL);
964 js_devdata = &kbdev->js_data;
965 js_kctx_info = &kctx->jctx.sched_info;
967 lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
968 lockdep_assert_held(&js_devdata->runpool_mutex);
970 /* Track total contexts */
971 --(js_devdata->nr_all_contexts_running);
972 KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running >= 0);
974 if ((js_kctx_info->ctx.flags & KBASE_CTX_FLAG_SUBMIT_DISABLED) == 0) {
975 /* Track contexts that can submit jobs */
976 --(js_devdata->nr_user_contexts_running);
977 KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running >= 0);
983 * @brief Submit atoms from all available contexts to all job slots.
985 * This will attempt to submit as many jobs as possible. It will exit when
986 * either all job slots are full, or all contexts have been used.
988 * @param[in] kbdev Device pointer
990 static inline void kbase_js_sched_all(struct kbase_device *kbdev)
992 kbase_js_sched(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1);
996 kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS];
998 extern const base_jd_prio
999 kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT];
1002 * kbasep_js_atom_prio_to_sched_prio(): - Convert atom priority (base_jd_prio)
1003 * to relative ordering
1004 * @atom_prio: Priority ID to translate.
1006 * Atom priority values for @ref base_jd_prio cannot be compared directly to
1007 * find out which are higher or lower.
1009 * This function will convert base_jd_prio values for successively lower
1010 * priorities into a monotonically increasing sequence. That is, the lower the
1011 * base_jd_prio priority, the higher the value produced by this function. This
1012 * is in accordance with how the rest of the kernel treates priority.
1014 * The mapping is 1:1 and the size of the valid input range is the same as the
1015 * size of the valid output range, i.e.
1016 * KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS
1018 * Note This must be kept in sync with BASE_JD_PRIO_<...> definitions
1020 * Return: On success: a value in the inclusive range
1021 * 0..KBASE_JS_ATOM_SCHED_PRIO_COUNT-1. On failure:
1022 * KBASE_JS_ATOM_SCHED_PRIO_INVALID
1024 static inline int kbasep_js_atom_prio_to_sched_prio(base_jd_prio atom_prio)
1026 if (atom_prio >= BASE_JD_NR_PRIO_LEVELS)
1027 return KBASE_JS_ATOM_SCHED_PRIO_INVALID;
1029 return kbasep_js_atom_priority_to_relative[atom_prio];
1032 static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(int sched_prio)
1034 unsigned int prio_idx;
1036 KBASE_DEBUG_ASSERT(0 <= sched_prio
1037 && sched_prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT);
1039 prio_idx = (unsigned int)sched_prio;
1041 return kbasep_js_relative_priority_to_atom[prio_idx];
1044 /** @} *//* end group kbase_js */
1045 /** @} *//* end group base_kbase_api */
1046 /** @} *//* end group base_api */
1048 #endif /* _KBASE_JS_H_ */