drivers/gpu/arm/midgard/r12p0_04rel0/backend/gpu/mali_kbase_jm_rb.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19 /*
  20  * Register-based HW access backend specific APIs
  21  */
  22
  23 #include <mali_kbase.h>
  24 #include <mali_kbase_hwaccess_jm.h>
  25 #include <mali_kbase_jm.h>
  26 #include <mali_kbase_js.h>
  27 #include <mali_kbase_10969_workaround.h>
  28 #include <backend/gpu/mali_kbase_device_internal.h>
  29 #include <backend/gpu/mali_kbase_jm_internal.h>
  30 #include <backend/gpu/mali_kbase_js_affinity.h>
  31 #include <backend/gpu/mali_kbase_js_internal.h>
  32 #include <backend/gpu/mali_kbase_pm_internal.h>
  33
  34 /* Return whether the specified ringbuffer is empty. HW access lock must be
  35  * held */
  36 #define SLOT_RB_EMPTY(rb)   (rb->write_idx == rb->read_idx)
  37 /* Return number of atoms currently in the specified ringbuffer. HW access lock
  38  * must be held */
  39 #define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx)
  40
  41 static void kbase_gpu_release_atom(struct kbase_device *kbdev,
  42                                         struct kbase_jd_atom *katom,
  43                                         ktime_t *end_timestamp);
  44
  45 /**
  46  * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer
  47  * @kbdev: Device pointer
  48  * @katom: Atom to enqueue
  49  *
  50  * Context: Caller must hold the HW access lock
  51  */
  52 static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev,
  53                                         struct kbase_jd_atom *katom)
  54 {
  55         struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr];
  56
  57         WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE);
  58
  59         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
  60
  61         rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom;
  62         rb->write_idx++;
  63
  64         katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
  65 }
  66
  67 /**
  68  * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once
  69  * it has been completed
  70  * @kbdev:         Device pointer
  71  * @js:            Job slot to remove atom from
  72  * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in
  73  *                 which case current time will be used.
  74  *
  75  * Context: Caller must hold the HW access lock
  76  *
  77  * Return: Atom removed from ringbuffer
  78  */
  79 static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev,
  80                                                 int js,
  81                                                 ktime_t *end_timestamp)
  82 {
  83         struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
  84         struct kbase_jd_atom *katom;
  85
  86         if (SLOT_RB_EMPTY(rb)) {
  87                 WARN(1, "GPU ringbuffer unexpectedly empty\n");
  88                 return NULL;
  89         }
  90
  91         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
  92
  93         katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom;
  94
  95         kbase_gpu_release_atom(kbdev, katom, end_timestamp);
  96
  97         rb->read_idx++;
  98
  99         katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB;
 100
 101         kbase_js_debug_log_current_affinities(kbdev);
 102
 103         return katom;
 104 }
 105
 106 struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js,
 107                                         int idx)
 108 {
 109         struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
 110
 111         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 112
 113         if ((SLOT_RB_ENTRIES(rb) - 1) < idx)
 114                 return NULL; /* idx out of range */
 115
 116         return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom;
 117 }
 118
 119 struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev,
 120                                         int js)
 121 {
 122         return kbase_gpu_inspect(kbdev, js, 0);
 123 }
 124
 125 struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev,
 126                                         int js)
 127 {
 128         struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
 129
 130         if (SLOT_RB_EMPTY(rb))
 131                 return NULL;
 132
 133         return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom;
 134 }
 135
 136 /**
 137  * kbase_gpu_atoms_submitted - Inspect whether a slot has any atoms currently
 138  * on the GPU
 139  * @kbdev:  Device pointer
 140  * @js:     Job slot to inspect
 141  *
 142  * Return: true if there are atoms on the GPU for slot js,
 143  *         false otherwise
 144  */
 145 static bool kbase_gpu_atoms_submitted(struct kbase_device *kbdev, int js)
 146 {
 147         int i;
 148
 149         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 150
 151         for (i = 0; i < SLOT_RB_SIZE; i++) {
 152                 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
 153
 154                 if (!katom)
 155                         return false;
 156                 if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED ||
 157                                 katom->gpu_rb_state == KBASE_ATOM_GPU_RB_READY)
 158                         return true;
 159         }
 160
 161         return false;
 162 }
 163
 164 /**
 165  * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms
 166  * currently on the GPU
 167  * @kbdev:  Device pointer
 168  *
 169  * Return: true if there are any atoms on the GPU, false otherwise
 170  */
 171 static bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev)
 172 {
 173         int js;
 174         int i;
 175
 176         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 177
 178         for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
 179                 for (i = 0; i < SLOT_RB_SIZE; i++) {
 180                         struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
 181
 182                         if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED)
 183                                 return true;
 184                 }
 185         }
 186         return false;
 187 }
 188
 189 int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js)
 190 {
 191         int nr = 0;
 192         int i;
 193
 194         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 195
 196         for (i = 0; i < SLOT_RB_SIZE; i++) {
 197                 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
 198
 199                 if (katom && (katom->gpu_rb_state ==
 200                                                 KBASE_ATOM_GPU_RB_SUBMITTED))
 201                         nr++;
 202         }
 203
 204         return nr;
 205 }
 206
 207 int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js)
 208 {
 209         int nr = 0;
 210         int i;
 211
 212         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 213
 214         for (i = 0; i < SLOT_RB_SIZE; i++) {
 215                 if (kbase_gpu_inspect(kbdev, js, i))
 216                         nr++;
 217         }
 218
 219         return nr;
 220 }
 221
 222 static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js,
 223                                 enum kbase_atom_gpu_rb_state min_rb_state)
 224 {
 225         int nr = 0;
 226         int i;
 227
 228         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 229
 230         for (i = 0; i < SLOT_RB_SIZE; i++) {
 231                 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
 232
 233                 if (katom && (katom->gpu_rb_state >= min_rb_state))
 234                         nr++;
 235         }
 236
 237         return nr;
 238 }
 239
 240 int kbase_backend_slot_free(struct kbase_device *kbdev, int js)
 241 {
 242         if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) !=
 243                                                 KBASE_RESET_GPU_NOT_PENDING) {
 244                 /* The GPU is being reset - so prevent submission */
 245                 return 0;
 246         }
 247
 248         return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js);
 249 }
 250
 251
 252 static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev,
 253                                                 struct kbase_jd_atom *katom);
 254
 255 static bool kbasep_js_job_check_ref_cores(struct kbase_device *kbdev,
 256                                                 int js,
 257                                                 struct kbase_jd_atom *katom)
 258 {
 259         /* The most recently checked affinity. Having this at this scope allows
 260          * us to guarantee that we've checked the affinity in this function
 261          * call.
 262          */
 263         u64 recently_chosen_affinity = 0;
 264         bool chosen_affinity = false;
 265         bool retry;
 266
 267         do {
 268                 retry = false;
 269
 270                 /* NOTE: The following uses a number of FALLTHROUGHs to optimize
 271                  * the calls to this function. Ending of the function is
 272                  * indicated by BREAK OUT */
 273                 switch (katom->coreref_state) {
 274                         /* State when job is first attempted to be run */
 275                 case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
 276                         KBASE_DEBUG_ASSERT(katom->affinity == 0);
 277
 278                         /* Compute affinity */
 279                         if (false == kbase_js_choose_affinity(
 280                                         &recently_chosen_affinity, kbdev, katom,
 281                                                                         js)) {
 282                                 /* No cores are currently available */
 283                                 /* *** BREAK OUT: No state transition *** */
 284                                 break;
 285                         }
 286
 287                         chosen_affinity = true;
 288
 289                         /* Request the cores */
 290                         kbase_pm_request_cores(kbdev,
 291                                         katom->core_req & BASE_JD_REQ_T,
 292                                                 recently_chosen_affinity);
 293
 294                         katom->affinity = recently_chosen_affinity;
 295
 296                         /* Proceed to next state */
 297                         katom->coreref_state =
 298                         KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
 299
 300                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 301
 302                 case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
 303                         {
 304                                 enum kbase_pm_cores_ready cores_ready;
 305
 306                                 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 307                                         (katom->core_req & BASE_JD_REQ_T));
 308
 309                                 cores_ready = kbase_pm_register_inuse_cores(
 310                                                 kbdev,
 311                                                 katom->core_req & BASE_JD_REQ_T,
 312                                                 katom->affinity);
 313                                 if (cores_ready == KBASE_NEW_AFFINITY) {
 314                                         /* Affinity no longer valid - return to
 315                                          * previous state */
 316                                         kbasep_js_job_check_deref_cores(kbdev,
 317                                                                         katom);
 318                                         KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 319                                         JS_CORE_REF_REGISTER_INUSE_FAILED,
 320                                                         katom->kctx, katom,
 321                                                         katom->jc, js,
 322                                                         (u32) katom->affinity);
 323                                         /* *** BREAK OUT: Return to previous
 324                                          * state, retry *** */
 325                                         retry = true;
 326                                         break;
 327                                 }
 328                                 if (cores_ready == KBASE_CORES_NOT_READY) {
 329                                         /* Stay in this state and return, to
 330                                          * retry at this state later */
 331                                         KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 332                                         JS_CORE_REF_REGISTER_INUSE_FAILED,
 333                                                         katom->kctx, katom,
 334                                                         katom->jc, js,
 335                                                         (u32) katom->affinity);
 336                                         /* *** BREAK OUT: No state transition
 337                                          * *** */
 338                                         break;
 339                                 }
 340                                 /* Proceed to next state */
 341                                 katom->coreref_state =
 342                                 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
 343                         }
 344
 345                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 346
 347                 case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
 348                         KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 349                                         (katom->core_req & BASE_JD_REQ_T));
 350
 351                         /* Optimize out choosing the affinity twice in the same
 352                          * function call */
 353                         if (chosen_affinity == false) {
 354                                 /* See if the affinity changed since a previous
 355                                  * call. */
 356                                 if (false == kbase_js_choose_affinity(
 357                                                 &recently_chosen_affinity,
 358                                                         kbdev, katom, js)) {
 359                                         /* No cores are currently available */
 360                                         kbasep_js_job_check_deref_cores(kbdev,
 361                                                                         katom);
 362                                         KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 363                                         JS_CORE_REF_REQUEST_ON_RECHECK_FAILED,
 364                                                 katom->kctx, katom,
 365                                                 katom->jc, js,
 366                                                 (u32) recently_chosen_affinity);
 367                                         /* *** BREAK OUT: Transition to lower
 368                                          * state *** */
 369                                         break;
 370                                 }
 371                                 chosen_affinity = true;
 372                         }
 373
 374                         /* Now see if this requires a different set of cores */
 375                         if (recently_chosen_affinity != katom->affinity) {
 376                                 enum kbase_pm_cores_ready cores_ready;
 377
 378                                 kbase_pm_request_cores(kbdev,
 379                                                 katom->core_req & BASE_JD_REQ_T,
 380                                                 recently_chosen_affinity);
 381
 382                                 /* Register new cores whilst we still hold the
 383                                  * old ones, to minimize power transitions */
 384                                 cores_ready =
 385                                         kbase_pm_register_inuse_cores(kbdev,
 386                                                 katom->core_req & BASE_JD_REQ_T,
 387                                                 recently_chosen_affinity);
 388                                 kbasep_js_job_check_deref_cores(kbdev, katom);
 389
 390                                 /* Fixup the state that was reduced by
 391                                  * deref_cores: */
 392                                 katom->coreref_state =
 393                                 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
 394                                 katom->affinity = recently_chosen_affinity;
 395                                 if (cores_ready == KBASE_NEW_AFFINITY) {
 396                                         /* Affinity no longer valid - return to
 397                                          * previous state */
 398                                         katom->coreref_state =
 399                                         KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
 400
 401                                         kbasep_js_job_check_deref_cores(kbdev,
 402                                                                         katom);
 403
 404                                         KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 405                                         JS_CORE_REF_REGISTER_INUSE_FAILED,
 406                                                         katom->kctx, katom,
 407                                                         katom->jc, js,
 408                                                         (u32) katom->affinity);
 409                                         /* *** BREAK OUT: Return to previous
 410                                          * state, retry *** */
 411                                         retry = true;
 412                                         break;
 413                                 }
 414                                 /* Now might be waiting for powerup again, with
 415                                  * a new affinity */
 416                                 if (cores_ready == KBASE_CORES_NOT_READY) {
 417                                         /* Return to previous state */
 418                                         katom->coreref_state =
 419                                         KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
 420                                         KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 421                                         JS_CORE_REF_REGISTER_ON_RECHECK_FAILED,
 422                                                         katom->kctx, katom,
 423                                                         katom->jc, js,
 424                                                         (u32) katom->affinity);
 425                                         /* *** BREAK OUT: Transition to lower
 426                                          * state *** */
 427                                         break;
 428                                 }
 429                         }
 430                         /* Proceed to next state */
 431                         katom->coreref_state =
 432                         KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS;
 433
 434                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 435                 case KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS:
 436                         KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 437                                         (katom->core_req & BASE_JD_REQ_T));
 438                         KBASE_DEBUG_ASSERT(katom->affinity ==
 439                                                 recently_chosen_affinity);
 440
 441                         /* Note: this is where the caller must've taken the
 442                          * runpool_irq.lock */
 443
 444                         /* Check for affinity violations - if there are any,
 445                          * then we just ask the caller to requeue and try again
 446                          * later */
 447                         if (kbase_js_affinity_would_violate(kbdev, js,
 448                                         katom->affinity) != false) {
 449                                 /* Return to previous state */
 450                                 katom->coreref_state =
 451                                 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
 452                                 /* *** BREAK OUT: Transition to lower state ***
 453                                  */
 454                                 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 455                                         JS_CORE_REF_AFFINITY_WOULD_VIOLATE,
 456                                         katom->kctx, katom, katom->jc, js,
 457                                         (u32) katom->affinity);
 458                                 break;
 459                         }
 460
 461                         /* No affinity violations would result, so the cores are
 462                          * ready */
 463                         katom->coreref_state = KBASE_ATOM_COREREF_STATE_READY;
 464                         /* *** BREAK OUT: Cores Ready *** */
 465                         break;
 466
 467                 default:
 468                         KBASE_DEBUG_ASSERT_MSG(false,
 469                                         "Unhandled kbase_atom_coreref_state %d",
 470                                                         katom->coreref_state);
 471                         break;
 472                 }
 473         } while (retry != false);
 474
 475         return (katom->coreref_state == KBASE_ATOM_COREREF_STATE_READY);
 476 }
 477
 478 static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev,
 479                                                 struct kbase_jd_atom *katom)
 480 {
 481         KBASE_DEBUG_ASSERT(kbdev != NULL);
 482         KBASE_DEBUG_ASSERT(katom != NULL);
 483
 484         switch (katom->coreref_state) {
 485         case KBASE_ATOM_COREREF_STATE_READY:
 486                 /* State where atom was submitted to the HW - just proceed to
 487                  * power-down */
 488                 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 489                                         (katom->core_req & BASE_JD_REQ_T));
 490
 491                 /* *** FALLTHROUGH *** */
 492
 493         case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
 494                 /* State where cores were registered */
 495                 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 496                                         (katom->core_req & BASE_JD_REQ_T));
 497                 kbase_pm_release_cores(kbdev, katom->core_req & BASE_JD_REQ_T,
 498                                                         katom->affinity);
 499
 500                 break;
 501
 502         case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
 503                 /* State where cores were requested, but not registered */
 504                 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 505                                         (katom->core_req & BASE_JD_REQ_T));
 506                 kbase_pm_unrequest_cores(kbdev, katom->core_req & BASE_JD_REQ_T,
 507                                                         katom->affinity);
 508                 break;
 509
 510         case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
 511                 /* Initial state - nothing required */
 512                 KBASE_DEBUG_ASSERT(katom->affinity == 0);
 513                 break;
 514
 515         default:
 516                 KBASE_DEBUG_ASSERT_MSG(false,
 517                                                 "Unhandled coreref_state: %d",
 518                                                         katom->coreref_state);
 519                 break;
 520         }
 521
 522         katom->affinity = 0;
 523         katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
 524 }
 525
 526 static void kbasep_js_job_check_deref_cores_nokatom(struct kbase_device *kbdev,
 527                 base_jd_core_req core_req, u64 affinity,
 528                 enum kbase_atom_coreref_state coreref_state)
 529 {
 530         KBASE_DEBUG_ASSERT(kbdev != NULL);
 531
 532         switch (coreref_state) {
 533         case KBASE_ATOM_COREREF_STATE_READY:
 534                 /* State where atom was submitted to the HW - just proceed to
 535                  * power-down */
 536                 KBASE_DEBUG_ASSERT(affinity != 0 ||
 537                                         (core_req & BASE_JD_REQ_T));
 538
 539                 /* *** FALLTHROUGH *** */
 540
 541         case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
 542                 /* State where cores were registered */
 543                 KBASE_DEBUG_ASSERT(affinity != 0 ||
 544                                         (core_req & BASE_JD_REQ_T));
 545                 kbase_pm_release_cores(kbdev, core_req & BASE_JD_REQ_T,
 546                                                         affinity);
 547
 548                 break;
 549
 550         case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
 551                 /* State where cores were requested, but not registered */
 552                 KBASE_DEBUG_ASSERT(affinity != 0 ||
 553                                         (core_req & BASE_JD_REQ_T));
 554                 kbase_pm_unrequest_cores(kbdev, core_req & BASE_JD_REQ_T,
 555                                                         affinity);
 556                 break;
 557
 558         case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
 559                 /* Initial state - nothing required */
 560                 KBASE_DEBUG_ASSERT(affinity == 0);
 561                 break;
 562
 563         default:
 564                 KBASE_DEBUG_ASSERT_MSG(false,
 565                                                 "Unhandled coreref_state: %d",
 566                                                         coreref_state);
 567                 break;
 568         }
 569 }
 570
 571 static void kbase_gpu_release_atom(struct kbase_device *kbdev,
 572                                         struct kbase_jd_atom *katom,
 573                                         ktime_t *end_timestamp)
 574 {
 575         switch (katom->gpu_rb_state) {
 576         case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
 577                 /* Should be impossible */
 578                 WARN(1, "Attempting to release atom not in ringbuffer\n");
 579                 break;
 580
 581         case KBASE_ATOM_GPU_RB_SUBMITTED:
 582                 /* Inform power management at start/finish of atom so it can
 583                  * update its GPU utilisation metrics. Mark atom as not
 584                  * submitted beforehand. */
 585                 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY;
 586                 kbase_pm_metrics_update(kbdev, end_timestamp);
 587
 588                 if (katom->core_req & BASE_JD_REQ_PERMON)
 589                         kbase_pm_release_gpu_cycle_counter(kbdev);
 590                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 591
 592         case KBASE_ATOM_GPU_RB_READY:
 593                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 594
 595         case KBASE_ATOM_GPU_RB_WAITING_SECURE_MODE:
 596                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 597
 598         case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
 599                 kbase_js_affinity_release_slot_cores(kbdev, katom->slot_nr,
 600                                                         katom->affinity);
 601                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 602
 603         case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
 604                 break;
 605
 606         case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
 607                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 608
 609         case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
 610                 break;
 611         }
 612
 613         katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
 614 }
 615
 616 static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev,
 617                                                 struct kbase_jd_atom *katom)
 618 {
 619         kbase_gpu_release_atom(kbdev, katom, NULL);
 620         katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS;
 621 }
 622
 623 static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js)
 624 {
 625         struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
 626         bool slot_busy[3];
 627
 628         if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
 629                 return true;
 630         slot_busy[0] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 0,
 631                                         KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
 632         slot_busy[1] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 1,
 633                                         KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
 634         slot_busy[2] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 2,
 635                                         KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
 636
 637         if ((js == 2 && !(slot_busy[0] || slot_busy[1])) ||
 638                 (js != 2 && !slot_busy[2]))
 639                 return true;
 640
 641         /* Don't submit slot 2 atom while GPU has jobs on slots 0/1 */
 642         if (js == 2 && (kbase_gpu_atoms_submitted(kbdev, 0) ||
 643                         kbase_gpu_atoms_submitted(kbdev, 1) ||
 644                         backend->rmu_workaround_flag))
 645                 return false;
 646
 647         /* Don't submit slot 0/1 atom while GPU has jobs on slot 2 */
 648         if (js != 2 && (kbase_gpu_atoms_submitted(kbdev, 2) ||
 649                         !backend->rmu_workaround_flag))
 650                 return false;
 651
 652         backend->rmu_workaround_flag = !backend->rmu_workaround_flag;
 653
 654         return true;
 655 }
 656
 657 static bool kbase_gpu_in_secure_mode(struct kbase_device *kbdev)
 658 {
 659         return kbdev->secure_mode;
 660 }
 661
 662 static int kbase_gpu_secure_mode_enable(struct kbase_device *kbdev)
 663 {
 664         int err = -EINVAL;
 665
 666         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 667
 668         WARN_ONCE(!kbdev->secure_ops,
 669                         "Cannot enable secure mode: secure callbacks not specified.\n");
 670
 671         if (kbdev->secure_ops) {
 672                 /* Switch GPU to secure mode */
 673                 err = kbdev->secure_ops->secure_mode_enable(kbdev);
 674
 675                 if (err)
 676                         dev_warn(kbdev->dev, "Failed to enable secure mode: %d\n", err);
 677                 else
 678                         kbdev->secure_mode = true;
 679         }
 680
 681         return err;
 682 }
 683
 684 static int kbase_gpu_secure_mode_disable(struct kbase_device *kbdev)
 685 {
 686         int err = -EINVAL;
 687
 688         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 689
 690         WARN_ONCE(!kbdev->secure_ops,
 691                         "Cannot disable secure mode: secure callbacks not specified.\n");
 692
 693         if (kbdev->secure_ops) {
 694                 /* Switch GPU to non-secure mode */
 695                 err = kbdev->secure_ops->secure_mode_disable(kbdev);
 696
 697                 if (err)
 698                         dev_warn(kbdev->dev, "Failed to disable secure mode: %d\n", err);
 699                 else
 700                         kbdev->secure_mode = false;
 701         }
 702
 703         return err;
 704 }
 705
 706 void kbase_gpu_slot_update(struct kbase_device *kbdev)
 707 {
 708         int js;
 709
 710         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 711
 712         for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
 713                 struct kbase_jd_atom *katom[2];
 714                 int idx;
 715
 716                 katom[0] = kbase_gpu_inspect(kbdev, js, 0);
 717                 katom[1] = kbase_gpu_inspect(kbdev, js, 1);
 718                 WARN_ON(katom[1] && !katom[0]);
 719
 720                 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
 721                         bool cores_ready;
 722
 723                         if (!katom[idx])
 724                                 continue;
 725
 726                         switch (katom[idx]->gpu_rb_state) {
 727                         case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
 728                                 /* Should be impossible */
 729                                 WARN(1, "Attempting to update atom not in ringbuffer\n");
 730                                 break;
 731
 732                         case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
 733                                 if (katom[idx]->atom_flags &
 734                                                 KBASE_KATOM_FLAG_X_DEP_BLOCKED)
 735                                         break;
 736
 737                                 katom[idx]->gpu_rb_state =
 738                                 KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE;
 739
 740                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 741                         case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
 742
 743                                 if (katom[idx]->will_fail_event_code) {
 744                                         kbase_gpu_mark_atom_for_return(kbdev,
 745                                                         katom[idx]);
 746                                         /* Set EVENT_DONE so this atom will be
 747                                            completed, not unpulled. */
 748                                         katom[idx]->event_code =
 749                                                 BASE_JD_EVENT_DONE;
 750                                         /* Only return if head atom or previous
 751                                          * atom already removed - as atoms must
 752                                          * be returned in order. */
 753                                         if (idx == 0 || katom[0]->gpu_rb_state ==
 754                                                         KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
 755                                                 kbase_gpu_dequeue_atom(kbdev, js, NULL);
 756                                                 kbase_jm_return_atom_to_js(kbdev, katom[idx]);
 757                                         }
 758                                         break;
 759                                 }
 760
 761
 762                                 cores_ready =
 763                                         kbasep_js_job_check_ref_cores(kbdev, js,
 764                                                                 katom[idx]);
 765
 766                                 if (katom[idx]->event_code ==
 767                                                 BASE_JD_EVENT_PM_EVENT) {
 768                                         katom[idx]->gpu_rb_state =
 769                                                 KBASE_ATOM_GPU_RB_RETURN_TO_JS;
 770                                         break;
 771                                 }
 772
 773                                 if (!cores_ready)
 774                                         break;
 775
 776                                 kbase_js_affinity_retain_slot_cores(kbdev, js,
 777                                                         katom[idx]->affinity);
 778                                 katom[idx]->gpu_rb_state =
 779                                         KBASE_ATOM_GPU_RB_WAITING_AFFINITY;
 780
 781                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 782
 783                         case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
 784                                 if (!kbase_gpu_rmu_workaround(kbdev, js))
 785                                         break;
 786
 787                                 katom[idx]->gpu_rb_state =
 788                                         KBASE_ATOM_GPU_RB_WAITING_SECURE_MODE;
 789
 790                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 791
 792                         case KBASE_ATOM_GPU_RB_WAITING_SECURE_MODE:
 793                                 /* Only submit if head atom or previous atom
 794                                  * already submitted */
 795                                 if (idx == 1 &&
 796                                         (katom[0]->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED &&
 797                                         katom[0]->gpu_rb_state != KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB))
 798                                         break;
 799
 800                                 if (kbase_gpu_in_secure_mode(kbdev) != kbase_jd_katom_is_secure(katom[idx])) {
 801                                         int err = 0;
 802
 803                                         /* Not in correct mode, take action */
 804                                         if (kbase_gpu_atoms_submitted_any(kbdev)) {
 805                                                 /*
 806                                                  * We are not in the correct
 807                                                  * GPU mode for this job, and
 808                                                  * we can't switch now because
 809                                                  * there are jobs already
 810                                                  * running.
 811                                                  */
 812                                                 break;
 813                                         }
 814
 815                                         /* No jobs running, so we can switch GPU mode right now */
 816                                         if (kbase_jd_katom_is_secure(katom[idx])) {
 817                                                 err = kbase_gpu_secure_mode_enable(kbdev);
 818                                         } else {
 819                                                 err = kbase_gpu_secure_mode_disable(kbdev);
 820                                         }
 821
 822                                         if (err) {
 823                                                 /* Failed to switch secure mode, fail atom */
 824                                                 katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID;
 825                                                 kbase_gpu_mark_atom_for_return(kbdev, katom[idx]);
 826                                                 /* Only return if head atom or previous atom
 827                                                  * already removed - as atoms must be returned
 828                                                  * in order */
 829                                                 if (idx == 0 || katom[0]->gpu_rb_state ==
 830                                                                 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
 831                                                         kbase_gpu_dequeue_atom(kbdev, js, NULL);
 832                                                         kbase_jm_return_atom_to_js(kbdev, katom[idx]);
 833                                                 }
 834                                                 break;
 835                                         }
 836                                 }
 837
 838                                 /* Secure mode sanity checks */
 839                                 KBASE_DEBUG_ASSERT_MSG(
 840                                         kbase_jd_katom_is_secure(katom[idx]) == kbase_gpu_in_secure_mode(kbdev),
 841                                         "Secure mode of atom (%d) doesn't match secure mode of GPU (%d)",
 842                                         kbase_jd_katom_is_secure(katom[idx]), kbase_gpu_in_secure_mode(kbdev));
 843                                 katom[idx]->gpu_rb_state =
 844                                         KBASE_ATOM_GPU_RB_READY;
 845
 846                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 847
 848                         case KBASE_ATOM_GPU_RB_READY:
 849                                 /* Only submit if head atom or previous atom
 850                                  * already submitted */
 851                                 if (idx == 1 &&
 852                                         (katom[0]->gpu_rb_state !=
 853                                                 KBASE_ATOM_GPU_RB_SUBMITTED &&
 854                                         katom[0]->gpu_rb_state !=
 855                                         KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB))
 856                                         break;
 857
 858                                 /* Check if this job needs the cycle counter
 859                                  * enabled before submission */
 860                                 if (katom[idx]->core_req & BASE_JD_REQ_PERMON)
 861                                         kbase_pm_request_gpu_cycle_counter_l2_is_on(
 862                                                                         kbdev);
 863
 864                                 kbase_job_hw_submit(kbdev, katom[idx], js);
 865                                 katom[idx]->gpu_rb_state =
 866                                                 KBASE_ATOM_GPU_RB_SUBMITTED;
 867
 868                                 /* Inform power management at start/finish of
 869                                  * atom so it can update its GPU utilisation
 870                                  * metrics. */
 871                                 kbase_pm_metrics_update(kbdev,
 872                                                 &katom[idx]->start_timestamp);
 873
 874                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 875
 876                         case KBASE_ATOM_GPU_RB_SUBMITTED:
 877                                 /* Atom submitted to HW, nothing else to do */
 878                                 break;
 879
 880                         case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
 881                                 /* Only return if head atom or previous atom
 882                                  * already removed - as atoms must be returned
 883                                  * in order */
 884                                 if (idx == 0 || katom[0]->gpu_rb_state ==
 885                                         KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
 886                                         kbase_gpu_dequeue_atom(kbdev, js, NULL);
 887                                         kbase_jm_return_atom_to_js(kbdev,
 888                                                                 katom[idx]);
 889                                 }
 890                                 break;
 891                         }
 892                 }
 893         }
 894
 895         /* Warn if PRLAM-8987 affinity restrictions are violated */
 896         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
 897                 WARN_ON((kbase_gpu_atoms_submitted(kbdev, 0) ||
 898                         kbase_gpu_atoms_submitted(kbdev, 1)) &&
 899                         kbase_gpu_atoms_submitted(kbdev, 2));
 900 }
 901
 902
 903 void kbase_backend_run_atom(struct kbase_device *kbdev,
 904                                 struct kbase_jd_atom *katom)
 905 {
 906         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 907
 908         kbase_gpu_enqueue_atom(kbdev, katom);
 909         kbase_gpu_slot_update(kbdev);
 910 }
 911
 912 bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js)
 913 {
 914         struct kbase_jd_atom *katom;
 915         struct kbase_jd_atom *next_katom;
 916
 917         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 918
 919         katom = kbase_gpu_inspect(kbdev, js, 0);
 920         next_katom = kbase_gpu_inspect(kbdev, js, 1);
 921
 922         if (next_katom && katom->kctx == next_katom->kctx &&
 923                 next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED &&
 924                 (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL)
 925                                                                         != 0 ||
 926                 kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL)
 927                                                                         != 0)) {
 928                 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
 929                                 JS_COMMAND_NOP, NULL);
 930                 next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY;
 931                 return true;
 932         }
 933
 934         return false;
 935 }
 936
 937 void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
 938                                 u32 completion_code,
 939                                 u64 job_tail,
 940                                 ktime_t *end_timestamp)
 941 {
 942         struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0);
 943         struct kbase_context *kctx = katom->kctx;
 944
 945         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 946
 947         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) &&
 948                         completion_code != BASE_JD_EVENT_DONE &&
 949                         !(completion_code & BASE_JD_SW_EVENT)) {
 950                 katom->need_cache_flush_cores_retained = katom->affinity;
 951                 kbase_pm_request_cores(kbdev, false, katom->affinity);
 952         } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) {
 953                 if (kbdev->gpu_props.num_core_groups > 1 &&
 954                         !(katom->affinity &
 955                         kbdev->gpu_props.props.coherency_info.group[0].core_mask
 956                                                                         ) &&
 957                         (katom->affinity &
 958                         kbdev->gpu_props.props.coherency_info.group[1].core_mask
 959                                                                         )) {
 960                         dev_info(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n");
 961                         katom->need_cache_flush_cores_retained =
 962                                                                 katom->affinity;
 963                         kbase_pm_request_cores(kbdev, false,
 964                                                         katom->affinity);
 965                 }
 966         }
 967
 968         katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp);
 969
 970         kbase_timeline_job_slot_done(kbdev, katom->kctx, katom, js, 0);
 971
 972         if (completion_code == BASE_JD_EVENT_STOPPED) {
 973                 struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
 974                                                                         0);
 975
 976                 /*
 977                  * Dequeue next atom from ringbuffers on same slot if required.
 978                  * This atom will already have been removed from the NEXT
 979                  * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that
 980                  * the atoms on this slot are returned in the correct order.
 981                  */
 982                 if (next_katom && katom->kctx == next_katom->kctx) {
 983                         kbase_gpu_dequeue_atom(kbdev, js, end_timestamp);
 984                         kbase_jm_return_atom_to_js(kbdev, next_katom);
 985                 }
 986         } else if (completion_code != BASE_JD_EVENT_DONE) {
 987                 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 988                 int i;
 989
 990 #if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0
 991                 KBASE_TRACE_DUMP(kbdev);
 992 #endif
 993                 kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
 994
 995                 /*
 996                  * Remove all atoms on the same context from ringbuffers. This
 997                  * will not remove atoms that are already on the GPU, as these
 998                  * are guaranteed not to have fail dependencies on the failed
 999                  * atom.
1000                  */
1001                 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) {
1002                         struct kbase_jd_atom *katom_idx0 =
1003                                                 kbase_gpu_inspect(kbdev, i, 0);
1004                         struct kbase_jd_atom *katom_idx1 =
1005                                                 kbase_gpu_inspect(kbdev, i, 1);
1006
1007                         if (katom_idx0 && katom_idx0->kctx == katom->kctx &&
1008                                 katom_idx0->gpu_rb_state !=
1009                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
1010                                 /* Dequeue katom_idx0 from ringbuffer */
1011                                 kbase_gpu_dequeue_atom(kbdev, i, end_timestamp);
1012
1013                                 if (katom_idx1 &&
1014                                         katom_idx1->kctx == katom->kctx &&
1015                                         katom_idx0->gpu_rb_state !=
1016                                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
1017                                         /* Dequeue katom_idx1 from ringbuffer */
1018                                         kbase_gpu_dequeue_atom(kbdev, i,
1019                                                         end_timestamp);
1020
1021                                         katom_idx1->event_code =
1022                                                         BASE_JD_EVENT_STOPPED;
1023                                         kbase_jm_return_atom_to_js(kbdev,
1024                                                                 katom_idx1);
1025                                 }
1026                                 katom_idx0->event_code = BASE_JD_EVENT_STOPPED;
1027                                 kbase_jm_return_atom_to_js(kbdev, katom_idx0);
1028
1029                         } else if (katom_idx1 &&
1030                                         katom_idx1->kctx == katom->kctx &&
1031                                         katom_idx1->gpu_rb_state !=
1032                                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
1033                                 /* Can not dequeue this atom yet - will be
1034                                  * dequeued when atom at idx0 completes */
1035                                 katom_idx1->event_code = BASE_JD_EVENT_STOPPED;
1036                                 kbase_gpu_mark_atom_for_return(kbdev,
1037                                                                 katom_idx1);
1038                         }
1039                 }
1040         }
1041
1042         KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc,
1043                                         js, completion_code);
1044
1045         if (job_tail != 0 && job_tail != katom->jc) {
1046                 bool was_updated = (job_tail != katom->jc);
1047
1048                 /* Some of the job has been executed, so we update the job chain
1049                  * address to where we should resume from */
1050                 katom->jc = job_tail;
1051                 if (was_updated)
1052                         KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx,
1053                                                 katom, job_tail, js);
1054         }
1055
1056         /* Only update the event code for jobs that weren't cancelled */
1057         if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED)
1058                 katom->event_code = (base_jd_event_code)completion_code;
1059
1060         kbase_device_trace_register_access(kctx, REG_WRITE,
1061                                                 JOB_CONTROL_REG(JOB_IRQ_CLEAR),
1062                                                 1 << js);
1063
1064         /* Complete the job, and start new ones
1065          *
1066          * Also defer remaining work onto the workqueue:
1067          * - Re-queue Soft-stopped jobs
1068          * - For any other jobs, queue the job back into the dependency system
1069          * - Schedule out the parent context if necessary, and schedule a new
1070          *   one in.
1071          */
1072 #ifdef CONFIG_GPU_TRACEPOINTS
1073         {
1074                 /* The atom in the HEAD */
1075                 struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
1076                                                                         0);
1077
1078                 if (next_katom && next_katom->gpu_rb_state ==
1079                                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
1080                         char js_string[16];
1081
1082                         trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
1083                                                                 js_string),
1084                                                 ktime_to_ns(*end_timestamp),
1085                                                 (u32)next_katom->kctx->id, 0,
1086                                                 next_katom->work_id);
1087                         kbdev->hwaccess.backend.slot_rb[js].last_context =
1088                                                         next_katom->kctx;
1089                 } else {
1090                         char js_string[16];
1091
1092                         trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
1093                                                                 js_string),
1094                                                 ktime_to_ns(ktime_get()), 0, 0,
1095                                                 0);
1096                         kbdev->hwaccess.backend.slot_rb[js].last_context = 0;
1097                 }
1098         }
1099 #endif
1100
1101         if (completion_code == BASE_JD_EVENT_STOPPED)
1102                 kbase_jm_return_atom_to_js(kbdev, katom);
1103         else
1104                 kbase_jm_complete(kbdev, katom, end_timestamp);
1105
1106         /* Job completion may have unblocked other atoms. Try to update all job
1107          * slots */
1108         kbase_gpu_slot_update(kbdev);
1109 }
1110
1111 void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp)
1112 {
1113         int js;
1114
1115         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1116
1117         for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1118                 int idx;
1119
1120                 for (idx = 0; idx < 2; idx++) {
1121                         struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
1122                                                                         js, 0);
1123
1124                         if (katom) {
1125                                 kbase_gpu_release_atom(kbdev, katom, NULL);
1126                                 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1127                                 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
1128                                 kbase_jm_complete(kbdev, katom, end_timestamp);
1129                         }
1130                 }
1131         }
1132 }
1133
1134 static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev,
1135                                         int js,
1136                                         struct kbase_jd_atom *katom,
1137                                         u32 action)
1138 {
1139         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1140         u32 hw_action = action & JS_COMMAND_MASK;
1141
1142         kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom);
1143         kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action,
1144                                                         katom->core_req, katom);
1145         kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
1146 }
1147
1148 static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev,
1149                                                 struct kbase_jd_atom *katom,
1150                                                 u32 action,
1151                                                 bool disjoint)
1152 {
1153         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1154
1155         katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;
1156         kbase_gpu_mark_atom_for_return(kbdev, katom);
1157         kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
1158
1159         if (disjoint)
1160                 kbase_job_check_enter_disjoint(kbdev, action, katom->core_req,
1161                                                                         katom);
1162 }
1163
1164 static int should_stop_x_dep_slot(struct kbase_jd_atom *katom)
1165 {
1166         if (katom->x_post_dep) {
1167                 struct kbase_jd_atom *dep_atom = katom->x_post_dep;
1168
1169                 if (dep_atom->gpu_rb_state !=
1170                                         KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB &&
1171                         dep_atom->gpu_rb_state !=
1172                                         KBASE_ATOM_GPU_RB_RETURN_TO_JS)
1173                         return dep_atom->slot_nr;
1174         }
1175         return -1;
1176 }
1177
1178 static void kbase_job_evicted(struct kbase_jd_atom *katom)
1179 {
1180         kbase_timeline_job_slot_done(katom->kctx->kbdev, katom->kctx, katom,
1181                         katom->slot_nr, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT);
1182 }
1183
1184 bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev,
1185                                         struct kbase_context *kctx,
1186                                         int js,
1187                                         struct kbase_jd_atom *katom,
1188                                         u32 action)
1189 {
1190         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1191
1192         struct kbase_jd_atom *katom_idx0;
1193         struct kbase_jd_atom *katom_idx1;
1194
1195         bool katom_idx0_valid, katom_idx1_valid;
1196
1197         bool ret = false;
1198
1199         int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1;
1200
1201         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1202
1203         katom_idx0 = kbase_gpu_inspect(kbdev, js, 0);
1204         katom_idx1 = kbase_gpu_inspect(kbdev, js, 1);
1205
1206         if (katom) {
1207                 katom_idx0_valid = (katom_idx0 == katom);
1208                 /* If idx0 is to be removed and idx1 is on the same context,
1209                  * then idx1 must also be removed otherwise the atoms might be
1210                  * returned out of order */
1211                 if (katom_idx1)
1212                         katom_idx1_valid = (katom_idx1 == katom) ||
1213                                                 (katom_idx0_valid &&
1214                                                         (katom_idx0->kctx ==
1215                                                         katom_idx1->kctx));
1216                 else
1217                         katom_idx1_valid = false;
1218         } else {
1219                 katom_idx0_valid = (katom_idx0 &&
1220                                         (!kctx || katom_idx0->kctx == kctx));
1221                 katom_idx1_valid = (katom_idx1 &&
1222                                         (!kctx || katom_idx1->kctx == kctx));
1223         }
1224
1225         if (katom_idx0_valid)
1226                 stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0);
1227         if (katom_idx1_valid)
1228                 stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1);
1229
1230         if (katom_idx0_valid) {
1231                 if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
1232                         /* Simple case - just dequeue and return */
1233                         kbase_gpu_dequeue_atom(kbdev, js, NULL);
1234                         if (katom_idx1_valid) {
1235                                 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1236                                 katom_idx1->event_code =
1237                                                 BASE_JD_EVENT_REMOVED_FROM_NEXT;
1238                                 kbase_jm_return_atom_to_js(kbdev, katom_idx1);
1239                                 kbasep_js_clear_submit_allowed(js_devdata,
1240                                                         katom_idx1->kctx);
1241                         }
1242
1243                         katom_idx0->event_code =
1244                                                 BASE_JD_EVENT_REMOVED_FROM_NEXT;
1245                         kbase_jm_return_atom_to_js(kbdev, katom_idx0);
1246                         kbasep_js_clear_submit_allowed(js_devdata,
1247                                                         katom_idx0->kctx);
1248                 } else {
1249                         /* katom_idx0 is on GPU */
1250                         if (katom_idx1 && katom_idx1->gpu_rb_state ==
1251                                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
1252                                 /* katom_idx0 and katom_idx1 are on GPU */
1253
1254                                 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1255                                                 JS_COMMAND_NEXT), NULL) == 0) {
1256                                         /* idx0 has already completed - stop
1257                                          * idx1 if needed*/
1258                                         if (katom_idx1_valid) {
1259                                                 kbase_gpu_stop_atom(kbdev, js,
1260                                                                 katom_idx1,
1261                                                                 action);
1262                                                 ret = true;
1263                                         }
1264                                 } else {
1265                                         /* idx1 is in NEXT registers - attempt
1266                                          * to remove */
1267                                         kbase_reg_write(kbdev,
1268                                                         JOB_SLOT_REG(js,
1269                                                         JS_COMMAND_NEXT),
1270                                                         JS_COMMAND_NOP, NULL);
1271
1272                                         if (kbase_reg_read(kbdev,
1273                                                         JOB_SLOT_REG(js,
1274                                                         JS_HEAD_NEXT_LO), NULL)
1275                                                                         != 0 ||
1276                                                 kbase_reg_read(kbdev,
1277                                                         JOB_SLOT_REG(js,
1278                                                         JS_HEAD_NEXT_HI), NULL)
1279                                                                         != 0) {
1280                                                 /* idx1 removed successfully,
1281                                                  * will be handled in IRQ */
1282                                                 kbase_job_evicted(katom_idx1);
1283                                                 kbase_gpu_remove_atom(kbdev,
1284                                                                 katom_idx1,
1285                                                                 action, true);
1286                                                 stop_x_dep_idx1 =
1287                                         should_stop_x_dep_slot(katom_idx1);
1288
1289                                                 /* stop idx0 if still on GPU */
1290                                                 kbase_gpu_stop_atom(kbdev, js,
1291                                                                 katom_idx0,
1292                                                                 action);
1293                                                 ret = true;
1294                                         } else if (katom_idx1_valid) {
1295                                                 /* idx0 has already completed,
1296                                                  * stop idx1 if needed */
1297                                                 kbase_gpu_stop_atom(kbdev, js,
1298                                                                 katom_idx1,
1299                                                                 action);
1300                                                 ret = true;
1301                                         }
1302                                 }
1303                         } else if (katom_idx1_valid) {
1304                                 /* idx1 not on GPU but must be dequeued*/
1305
1306                                 /* idx1 will be handled in IRQ */
1307                                 kbase_gpu_remove_atom(kbdev, katom_idx1, action,
1308                                                                         false);
1309                                 /* stop idx0 */
1310                                 /* This will be repeated for anything removed
1311                                  * from the next registers, since their normal
1312                                  * flow was also interrupted, and this function
1313                                  * might not enter disjoint state e.g. if we
1314                                  * don't actually do a hard stop on the head
1315                                  * atom */
1316                                 kbase_gpu_stop_atom(kbdev, js, katom_idx0,
1317                                                                         action);
1318                                 ret = true;
1319                         } else {
1320                                 /* no atom in idx1 */
1321                                 /* just stop idx0 */
1322                                 kbase_gpu_stop_atom(kbdev, js, katom_idx0,
1323                                                                         action);
1324                                 ret = true;
1325                         }
1326                 }
1327         } else if (katom_idx1_valid) {
1328                 if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
1329                         /* Mark for return */
1330                         /* idx1 will be returned once idx0 completes */
1331                         kbase_gpu_remove_atom(kbdev, katom_idx1, action,
1332                                                                         false);
1333                 } else {
1334                         /* idx1 is on GPU */
1335                         if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1336                                                 JS_COMMAND_NEXT), NULL) == 0) {
1337                                 /* idx0 has already completed - stop idx1 */
1338                                 kbase_gpu_stop_atom(kbdev, js, katom_idx1,
1339                                                                         action);
1340                                 ret = true;
1341                         } else {
1342                                 /* idx1 is in NEXT registers - attempt to
1343                                  * remove */
1344                                 kbase_reg_write(kbdev, JOB_SLOT_REG(js,
1345                                                         JS_COMMAND_NEXT),
1346                                                         JS_COMMAND_NOP, NULL);
1347
1348                                 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1349                                                 JS_HEAD_NEXT_LO), NULL) != 0 ||
1350                                     kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1351                                                 JS_HEAD_NEXT_HI), NULL) != 0) {
1352                                         /* idx1 removed successfully, will be
1353                                          * handled in IRQ once idx0 completes */
1354                                         kbase_job_evicted(katom_idx1);
1355                                         kbase_gpu_remove_atom(kbdev, katom_idx1,
1356                                                                         action,
1357                                                                         false);
1358                                 } else {
1359                                         /* idx0 has already completed - stop
1360                                          * idx1 */
1361                                         kbase_gpu_stop_atom(kbdev, js,
1362                                                                 katom_idx1,
1363                                                                 action);
1364                                         ret = true;
1365                                 }
1366                         }
1367                 }
1368         }
1369
1370
1371         if (stop_x_dep_idx0 != -1)
1372                 kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0,
1373                                                                 NULL, action);
1374
1375         if (stop_x_dep_idx1 != -1)
1376                 kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1,
1377                                                                 NULL, action);
1378
1379         return ret;
1380 }
1381
1382 void kbase_gpu_cacheclean(struct kbase_device *kbdev,
1383                                         struct kbase_jd_atom *katom)
1384 {
1385         /* Limit the number of loops to avoid a hang if the interrupt is missed
1386          */
1387         u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS;
1388
1389         mutex_lock(&kbdev->cacheclean_lock);
1390
1391         /* use GPU_COMMAND completion solution */
1392         /* clean & invalidate the caches */
1393         KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
1394         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1395                                         GPU_COMMAND_CLEAN_INV_CACHES, NULL);
1396
1397         /* wait for cache flush to complete before continuing */
1398         while (--max_loops &&
1399                 (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) &
1400                                                 CLEAN_CACHES_COMPLETED) == 0)
1401                 ;
1402
1403         /* clear the CLEAN_CACHES_COMPLETED irq */
1404         KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u,
1405                                                         CLEAN_CACHES_COMPLETED);
1406         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR),
1407                                                 CLEAN_CACHES_COMPLETED, NULL);
1408         KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.backend.state !=
1409                                                 KBASE_INSTR_STATE_CLEANING,
1410             "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang.");
1411
1412         mutex_unlock(&kbdev->cacheclean_lock);
1413
1414         kbase_pm_unrequest_cores(kbdev, false,
1415                                         katom->need_cache_flush_cores_retained);
1416 }
1417
1418 void kbase_backend_complete_wq(struct kbase_device *kbdev,
1419                                                 struct kbase_jd_atom *katom)
1420 {
1421         /*
1422          * If cache flush required due to HW workaround then perform the flush
1423          * now
1424          */
1425         if (katom->need_cache_flush_cores_retained) {
1426                 kbase_gpu_cacheclean(kbdev, katom);
1427                 katom->need_cache_flush_cores_retained = 0;
1428         }
1429
1430         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969)            &&
1431             (katom->core_req & BASE_JD_REQ_FS)                        &&
1432             katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT       &&
1433             (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) &&
1434             !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) {
1435                 dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n");
1436                 if (kbasep_10969_workaround_clamp_coordinates(katom)) {
1437                         /* The job had a TILE_RANGE_FAULT after was soft-stopped
1438                          * Due to an HW issue we try to execute the job again.
1439                          */
1440                         dev_dbg(kbdev->dev,
1441                                 "Clamping has been executed, try to rerun the job\n"
1442                         );
1443                         katom->event_code = BASE_JD_EVENT_STOPPED;
1444                         katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN;
1445                 }
1446         }
1447
1448         /* Clear the coreref_state now - while check_deref_cores() may not have
1449          * been called yet, the caller will have taken a copy of this field. If
1450          * this is not done, then if the atom is re-scheduled (following a soft
1451          * stop) then the core reference would not be retaken. */
1452         katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
1453         katom->affinity = 0;
1454 }
1455
1456 void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev,
1457                 base_jd_core_req core_req, u64 affinity,
1458                 enum kbase_atom_coreref_state coreref_state)
1459 {
1460         kbasep_js_job_check_deref_cores_nokatom(kbdev, core_req, affinity,
1461                         coreref_state);
1462
1463         if (!kbdev->pm.active_count) {
1464                 mutex_lock(&kbdev->js_data.runpool_mutex);
1465                 mutex_lock(&kbdev->pm.lock);
1466                 kbase_pm_update_active(kbdev);
1467                 mutex_unlock(&kbdev->pm.lock);
1468                 mutex_unlock(&kbdev->js_data.runpool_mutex);
1469         }
1470 }
1471
1472 void kbase_gpu_dump_slots(struct kbase_device *kbdev)
1473 {
1474         struct kbasep_js_device_data *js_devdata;
1475         unsigned long flags;
1476         int js;
1477
1478         js_devdata = &kbdev->js_data;
1479
1480         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1481
1482         dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n");
1483
1484         for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1485                 int idx;
1486
1487                 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
1488                         struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
1489                                                                         js,
1490                                                                         idx);
1491
1492                         if (katom)
1493                                 dev_info(kbdev->dev,
1494                                 "  js%d idx%d : katom=%p gpu_rb_state=%d\n",
1495                                 js, idx, katom, katom->gpu_rb_state);
1496                         else
1497                                 dev_info(kbdev->dev, "  js%d idx%d : empty\n",
1498                                                                 js, idx);
1499                 }
1500         }
1501
1502         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1503 }
1504
1505
1506