drivers/gpu/arm/mali400/mali/common/mali_group.c

   1 /*
   2  * Copyright (C) 2011-2012 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_kernel_common.h"
  12 #include "mali_group.h"
  13 #include "mali_osk.h"
  14 #include "mali_l2_cache.h"
  15 #include "mali_gp.h"
  16 #include "mali_pp.h"
  17 #include "mali_mmu.h"
  18 #include "mali_dlbu.h"
  19 #include "mali_broadcast.h"
  20 #include "mali_gp_scheduler.h"
  21 #include "mali_pp_scheduler.h"
  22 #include "mali_kernel_core.h"
  23 #include "mali_osk_profiling.h"
  24
  25 static void mali_group_bottom_half_mmu(void *data);
  26 static void mali_group_bottom_half_gp(void *data);
  27 static void mali_group_bottom_half_pp(void *data);
  28
  29 static void mali_group_timeout(void *data);
  30 static void mali_group_reset_pp(struct mali_group *group);
  31
  32 #if defined(CONFIG_MALI400_PROFILING)
  33 static void mali_group_report_l2_cache_counters_per_core(struct mali_group *group, u32 core_num);
  34 #endif /* #if defined(CONFIG_MALI400_PROFILING) */
  35
  36 /*
  37  * The group object is the most important object in the device driver,
  38  * and acts as the center of many HW operations.
  39  * The reason for this is that operations on the MMU will affect all
  40  * cores connected to this MMU (a group is defined by the MMU and the
  41  * cores which are connected to this).
  42  * The group lock is thus the most important lock, followed by the
  43  * GP and PP scheduler locks. They must be taken in the following
  44  * order:
  45  * GP/PP lock first, then group lock(s).
  46  */
  47
  48 static struct mali_group *mali_global_groups[MALI_MAX_NUMBER_OF_GROUPS];
  49 static u32 mali_global_num_groups = 0;
  50
  51 enum mali_group_activate_pd_status
  52 {
  53         MALI_GROUP_ACTIVATE_PD_STATUS_FAILED,
  54         MALI_GROUP_ACTIVATE_PD_STATUS_OK_KEPT_PD,
  55         MALI_GROUP_ACTIVATE_PD_STATUS_OK_SWITCHED_PD,
  56 };
  57
  58 /* local helper functions */
  59 static enum mali_group_activate_pd_status mali_group_activate_page_directory(struct mali_group *group, struct mali_session_data *session);
  60 static void mali_group_deactivate_page_directory(struct mali_group *group, struct mali_session_data *session);
  61 static void mali_group_remove_session_if_unused(struct mali_group *group, struct mali_session_data *session);
  62 static void mali_group_recovery_reset(struct mali_group *group);
  63 static void mali_group_mmu_page_fault(struct mali_group *group);
  64
  65 static void mali_group_post_process_job_pp(struct mali_group *group);
  66 static void mali_group_post_process_job_gp(struct mali_group *group, mali_bool suspend);
  67
  68 void mali_group_lock(struct mali_group *group)
  69 {
  70         if(_MALI_OSK_ERR_OK != _mali_osk_lock_wait(group->lock, _MALI_OSK_LOCKMODE_RW))
  71         {
  72                 /* Non-interruptable lock failed: this should never happen. */
  73                 MALI_DEBUG_ASSERT(0);
  74         }
  75         MALI_DEBUG_PRINT(5, ("Mali group: Group lock taken 0x%08X\n", group));
  76 }
  77
  78 void mali_group_unlock(struct mali_group *group)
  79 {
  80         MALI_DEBUG_PRINT(5, ("Mali group: Releasing group lock 0x%08X\n", group));
  81         _mali_osk_lock_signal(group->lock, _MALI_OSK_LOCKMODE_RW);
  82 }
  83
  84 #ifdef DEBUG
  85 void mali_group_assert_locked(struct mali_group *group)
  86 {
  87         MALI_DEBUG_ASSERT_LOCK_HELD(group->lock);
  88 }
  89 #endif
  90
  91
  92 struct mali_group *mali_group_create(struct mali_l2_cache_core *core, struct mali_dlbu_core *dlbu, struct mali_bcast_unit *bcast)
  93 {
  94         struct mali_group *group = NULL;
  95         _mali_osk_lock_flags_t lock_flags;
  96
  97 #if defined(MALI_UPPER_HALF_SCHEDULING)
  98         lock_flags = _MALI_OSK_LOCKFLAG_ORDERED | _MALI_OSK_LOCKFLAG_SPINLOCK_IRQ | _MALI_OSK_LOCKFLAG_NONINTERRUPTABLE;
  99 #else
 100         lock_flags = _MALI_OSK_LOCKFLAG_ORDERED | _MALI_OSK_LOCKFLAG_SPINLOCK | _MALI_OSK_LOCKFLAG_NONINTERRUPTABLE;
 101 #endif
 102
 103         if (mali_global_num_groups >= MALI_MAX_NUMBER_OF_GROUPS)
 104         {
 105                 MALI_PRINT_ERROR(("Mali group: Too many group objects created\n"));
 106                 return NULL;
 107         }
 108
 109         group = _mali_osk_calloc(1, sizeof(struct mali_group));
 110         if (NULL != group)
 111         {
 112                 group->timeout_timer = _mali_osk_timer_init();
 113
 114                 if (NULL != group->timeout_timer)
 115                 {
 116                         _mali_osk_lock_order_t order;
 117                         _mali_osk_timer_setcallback(group->timeout_timer, mali_group_timeout, (void *)group);
 118
 119                         if (NULL != dlbu)
 120                         {
 121                                 order = _MALI_OSK_LOCK_ORDER_GROUP_VIRTUAL;
 122                         }
 123                         else
 124                         {
 125                                 order = _MALI_OSK_LOCK_ORDER_GROUP;
 126                         }
 127
 128                         group->lock = _mali_osk_lock_init(lock_flags, 0, order);
 129                         if (NULL != group->lock)
 130                         {
 131                                 group->l2_cache_core[0] = core;
 132                                 group->session = NULL;
 133                                 group->page_dir_ref_count = 0;
 134                                 group->power_is_on = MALI_TRUE;
 135                                 group->state = MALI_GROUP_STATE_IDLE;
 136                                 _mali_osk_list_init(&group->group_list);
 137                                 _mali_osk_list_init(&group->pp_scheduler_list);
 138                                 group->parent_group = NULL;
 139                                 group->l2_cache_core_ref_count[0] = 0;
 140                                 group->l2_cache_core_ref_count[1] = 0;
 141                                 group->bcast_core = bcast;
 142                                 group->dlbu_core = dlbu;
 143
 144                                 mali_global_groups[mali_global_num_groups] = group;
 145                                 mali_global_num_groups++;
 146
 147                                 return group;
 148                         }
 149             _mali_osk_timer_term(group->timeout_timer);
 150                 }
 151                 _mali_osk_free(group);
 152         }
 153
 154         return NULL;
 155 }
 156
 157 _mali_osk_errcode_t mali_group_add_mmu_core(struct mali_group *group, struct mali_mmu_core* mmu_core)
 158 {
 159         /* This group object now owns the MMU core object */
 160         group->mmu= mmu_core;
 161         group->bottom_half_work_mmu = _mali_osk_wq_create_work(mali_group_bottom_half_mmu, group);
 162         if (NULL == group->bottom_half_work_mmu)
 163         {
 164                 return _MALI_OSK_ERR_FAULT;
 165         }
 166         return _MALI_OSK_ERR_OK;
 167 }
 168
 169 void mali_group_remove_mmu_core(struct mali_group *group)
 170 {
 171         /* This group object no longer owns the MMU core object */
 172         group->mmu = NULL;
 173         if (NULL != group->bottom_half_work_mmu)
 174         {
 175                 _mali_osk_wq_delete_work(group->bottom_half_work_mmu);
 176         }
 177 }
 178
 179 _mali_osk_errcode_t mali_group_add_gp_core(struct mali_group *group, struct mali_gp_core* gp_core)
 180 {
 181         /* This group object now owns the GP core object */
 182         group->gp_core = gp_core;
 183         group->bottom_half_work_gp = _mali_osk_wq_create_work(mali_group_bottom_half_gp, group);
 184         if (NULL == group->bottom_half_work_gp)
 185         {
 186                 return _MALI_OSK_ERR_FAULT;
 187         }
 188         return _MALI_OSK_ERR_OK;
 189 }
 190
 191 void mali_group_remove_gp_core(struct mali_group *group)
 192 {
 193         /* This group object no longer owns the GP core object */
 194         group->gp_core = NULL;
 195         if (NULL != group->bottom_half_work_gp)
 196         {
 197                 _mali_osk_wq_delete_work(group->bottom_half_work_gp);
 198         }
 199 }
 200
 201 _mali_osk_errcode_t mali_group_add_pp_core(struct mali_group *group, struct mali_pp_core* pp_core)
 202 {
 203         /* This group object now owns the PP core object */
 204         group->pp_core = pp_core;
 205         group->bottom_half_work_pp = _mali_osk_wq_create_work(mali_group_bottom_half_pp, group);
 206         if (NULL == group->bottom_half_work_pp)
 207         {
 208                 return _MALI_OSK_ERR_FAULT;
 209         }
 210         return _MALI_OSK_ERR_OK;
 211 }
 212
 213 void mali_group_remove_pp_core(struct mali_group *group)
 214 {
 215         /* This group object no longer owns the PP core object */
 216         group->pp_core = NULL;
 217         if (NULL != group->bottom_half_work_pp)
 218         {
 219                 _mali_osk_wq_delete_work(group->bottom_half_work_pp);
 220         }
 221 }
 222
 223 void mali_group_delete(struct mali_group *group)
 224 {
 225         u32 i;
 226
 227         MALI_DEBUG_PRINT(4, ("Deleting group %p\n", group));
 228
 229         MALI_DEBUG_ASSERT(NULL == group->parent_group);
 230
 231         /* Delete the resources that this group owns */
 232         if (NULL != group->gp_core)
 233         {
 234                 mali_gp_delete(group->gp_core);
 235         }
 236
 237         if (NULL != group->pp_core)
 238         {
 239                 mali_pp_delete(group->pp_core);
 240         }
 241
 242         if (NULL != group->mmu)
 243         {
 244                 mali_mmu_delete(group->mmu);
 245         }
 246
 247         if (mali_group_is_virtual(group))
 248         {
 249                 /* Remove all groups from virtual group */
 250                 struct mali_group *child;
 251                 struct mali_group *temp;
 252
 253                 _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list)
 254                 {
 255                         child->parent_group = NULL;
 256                         mali_group_delete(child);
 257                 }
 258
 259                 mali_dlbu_delete(group->dlbu_core);
 260
 261                 if (NULL != group->bcast_core)
 262                 {
 263                         mali_bcast_unit_delete(group->bcast_core);
 264                 }
 265         }
 266
 267         for (i = 0; i < MALI_MAX_NUMBER_OF_GROUPS; i++)
 268         {
 269                 if (mali_global_groups[i] == group)
 270                 {
 271                         mali_global_groups[i] = NULL;
 272                         mali_global_num_groups--;
 273
 274                         if (i != mali_global_num_groups)
 275                         {
 276                                 /* We removed a group from the middle of the array -- move the last
 277                                  * group to the current position to close the gap */
 278                                 mali_global_groups[i] = mali_global_groups[mali_global_num_groups];
 279                                 mali_global_groups[mali_global_num_groups] = NULL;
 280                         }
 281
 282                         break;
 283                 }
 284         }
 285
 286         if (NULL != group->timeout_timer)
 287         {
 288                 _mali_osk_timer_del(group->timeout_timer);
 289                 _mali_osk_timer_term(group->timeout_timer);
 290         }
 291
 292         if (NULL != group->bottom_half_work_mmu)
 293         {
 294                 _mali_osk_wq_delete_work(group->bottom_half_work_mmu);
 295         }
 296
 297         if (NULL != group->bottom_half_work_gp)
 298         {
 299                 _mali_osk_wq_delete_work(group->bottom_half_work_gp);
 300         }
 301
 302         if (NULL != group->bottom_half_work_pp)
 303         {
 304                 _mali_osk_wq_delete_work(group->bottom_half_work_pp);
 305         }
 306
 307         _mali_osk_lock_term(group->lock);
 308
 309         _mali_osk_free(group);
 310 }
 311
 312 MALI_DEBUG_CODE(static void mali_group_print_virtual(struct mali_group *vgroup)
 313 {
 314         u32 i;
 315         struct mali_group *group;
 316         struct mali_group *temp;
 317
 318         MALI_DEBUG_PRINT(4, ("Virtual group %p\n", vgroup));
 319         MALI_DEBUG_PRINT(4, ("l2_cache_core[0] = %p, ref = %d\n", vgroup->l2_cache_core[0], vgroup->l2_cache_core_ref_count[0]));
 320         MALI_DEBUG_PRINT(4, ("l2_cache_core[1] = %p, ref = %d\n", vgroup->l2_cache_core[1], vgroup->l2_cache_core_ref_count[1]));
 321
 322         i = 0;
 323         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &vgroup->group_list, struct mali_group, group_list)
 324         {
 325                 MALI_DEBUG_PRINT(4, ("[%d] %p, l2_cache_core[0] = %p\n", i, group, group->l2_cache_core[0]));
 326                 i++;
 327         }
 328 })
 329
 330 /**
 331  * @brief Add child group to virtual group parent
 332  *
 333  * Before calling this function, child must have it's state set to JOINING_VIRTUAL
 334  * to ensure it's not touched during the transition period. When this function returns,
 335  * child's state will be IN_VIRTUAL.
 336  */
 337 void mali_group_add_group(struct mali_group *parent, struct mali_group *child)
 338 {
 339         mali_bool found;
 340         u32 i;
 341
 342         MALI_DEBUG_PRINT(3, ("Adding group %p to virtual group %p\n", child, parent));
 343
 344         MALI_ASSERT_GROUP_LOCKED(parent);
 345
 346         MALI_DEBUG_ASSERT(mali_group_is_virtual(parent));
 347         MALI_DEBUG_ASSERT(!mali_group_is_virtual(child));
 348         MALI_DEBUG_ASSERT(NULL == child->parent_group);
 349         MALI_DEBUG_ASSERT(MALI_GROUP_STATE_JOINING_VIRTUAL == child->state);
 350
 351         _mali_osk_list_addtail(&child->group_list, &parent->group_list);
 352
 353         child->state = MALI_GROUP_STATE_IN_VIRTUAL;
 354         child->parent_group = parent;
 355
 356         MALI_DEBUG_ASSERT_POINTER(child->l2_cache_core[0]);
 357
 358         MALI_DEBUG_PRINT(4, ("parent->l2_cache_core: [0] = %p, [1] = %p\n", parent->l2_cache_core[0], parent->l2_cache_core[1]));
 359         MALI_DEBUG_PRINT(4, ("child->l2_cache_core: [0] = %p, [1] = %p\n", child->l2_cache_core[0], child->l2_cache_core[1]));
 360
 361         /* Keep track of the L2 cache cores of child groups */
 362         found = MALI_FALSE;
 363         for (i = 0; i < 2; i++)
 364         {
 365                 if (parent->l2_cache_core[i] == child->l2_cache_core[0])
 366                 {
 367                         MALI_DEBUG_ASSERT(parent->l2_cache_core_ref_count[i] > 0);
 368                         parent->l2_cache_core_ref_count[i]++;
 369                         found = MALI_TRUE;
 370                 }
 371         }
 372
 373         if (!found)
 374         {
 375                 /* First time we see this L2 cache, add it to our list */
 376                 i = (NULL == parent->l2_cache_core[0]) ? 0 : 1;
 377
 378                 MALI_DEBUG_PRINT(4, ("First time we see l2_cache %p. Adding to [%d] = %p\n", child->l2_cache_core[0], i, parent->l2_cache_core[i]));
 379
 380                 MALI_DEBUG_ASSERT(NULL == parent->l2_cache_core[i]);
 381
 382                 parent->l2_cache_core[i] = child->l2_cache_core[0];
 383                 parent->l2_cache_core_ref_count[i]++;
 384         }
 385
 386         /* Update Broadcast Unit and DLBU */
 387         mali_bcast_add_group(parent->bcast_core, child);
 388         mali_dlbu_add_group(parent->dlbu_core, child);
 389
 390         /* Update MMU */
 391         MALI_DEBUG_ASSERT(0 == child->page_dir_ref_count);
 392         if (parent->session == child->session)
 393         {
 394                 mali_mmu_zap_tlb(child->mmu);
 395         }
 396         else
 397         {
 398                 child->session = NULL;
 399
 400                 if (NULL == parent->session)
 401                 {
 402                         mali_mmu_activate_empty_page_directory(child->mmu);
 403                 }
 404                 else
 405                 {
 406
 407                         mali_bool activate_success = mali_mmu_activate_page_directory(child->mmu,
 408                                 mali_session_get_page_directory(parent->session));
 409                         MALI_DEBUG_ASSERT(activate_success);
 410                         MALI_IGNORE(activate_success);
 411                 }
 412         }
 413         child->session = NULL;
 414
 415         /* Start job on child when parent is active */
 416         if (NULL != parent->pp_running_job)
 417         {
 418                 struct mali_pp_job *job = parent->pp_running_job;
 419                 MALI_DEBUG_PRINT(3, ("Group %x joining running job %d on virtual group %x\n",
 420                                      child, mali_pp_job_get_id(job), parent));
 421                 MALI_DEBUG_ASSERT(MALI_GROUP_STATE_WORKING == parent->state);
 422                 mali_pp_job_start(child->pp_core, job, mali_pp_core_get_id(child->pp_core), MALI_TRUE);
 423
 424                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE|
 425                                               MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core))|
 426                                               MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH,
 427                                               mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job), 0, 0, 0);
 428
 429                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START|
 430                                               MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core))|
 431                                               MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL,
 432                                               mali_pp_job_get_pid(job), mali_pp_job_get_tid(job), 0, 0, 0);
 433         }
 434
 435         MALI_DEBUG_CODE(mali_group_print_virtual(parent);)
 436 }
 437
 438 /**
 439  * @brief Remove child group from virtual group parent
 440  *
 441  * After the child is removed, it's state will be LEAVING_VIRTUAL and must be set
 442  * to IDLE before it can be used.
 443  */
 444 void mali_group_remove_group(struct mali_group *parent, struct mali_group *child)
 445 {
 446         u32 i;
 447
 448         MALI_ASSERT_GROUP_LOCKED(parent);
 449
 450         MALI_DEBUG_PRINT(3, ("Removing group %p from virtual group %p\n", child, parent));
 451
 452         MALI_DEBUG_ASSERT(mali_group_is_virtual(parent));
 453         MALI_DEBUG_ASSERT(!mali_group_is_virtual(child));
 454         MALI_DEBUG_ASSERT(parent == child->parent_group);
 455         MALI_DEBUG_ASSERT(MALI_GROUP_STATE_IN_VIRTUAL == child->state);
 456         /* Removing groups while running is not yet supported. */
 457         MALI_DEBUG_ASSERT(MALI_GROUP_STATE_IDLE == parent->state);
 458
 459         mali_group_lock(child);
 460
 461         /* Update Broadcast Unit and DLBU */
 462         mali_bcast_remove_group(parent->bcast_core, child);
 463         mali_dlbu_remove_group(parent->dlbu_core, child);
 464
 465         _mali_osk_list_delinit(&child->group_list);
 466
 467         child->session = parent->session;
 468         child->parent_group = NULL;
 469         child->state = MALI_GROUP_STATE_LEAVING_VIRTUAL;
 470
 471         /* Keep track of the L2 cache cores of child groups */
 472         i = (child->l2_cache_core[0] == parent->l2_cache_core[0]) ? 0 : 1;
 473
 474         MALI_DEBUG_ASSERT(child->l2_cache_core[0] == parent->l2_cache_core[i]);
 475
 476         parent->l2_cache_core_ref_count[i]--;
 477
 478         if (parent->l2_cache_core_ref_count[i] == 0)
 479         {
 480                 parent->l2_cache_core[i] = NULL;
 481         }
 482
 483         MALI_DEBUG_CODE(mali_group_print_virtual(parent));
 484
 485         mali_group_unlock(child);
 486 }
 487
 488 struct mali_group *mali_group_acquire_group(struct mali_group *parent)
 489 {
 490         struct mali_group *child;
 491
 492         MALI_ASSERT_GROUP_LOCKED(parent);
 493
 494         MALI_DEBUG_ASSERT(mali_group_is_virtual(parent));
 495         MALI_DEBUG_ASSERT(!_mali_osk_list_empty(&parent->group_list));
 496
 497         child = _MALI_OSK_LIST_ENTRY(parent->group_list.prev, struct mali_group, group_list);
 498
 499         mali_group_remove_group(parent, child);
 500
 501         return child;
 502 }
 503
 504 void mali_group_reset(struct mali_group *group)
 505 {
 506         /*
 507          * This function should not be used to abort jobs,
 508          * currently only called during insmod and PM resume
 509          */
 510         MALI_DEBUG_ASSERT(NULL == group->gp_running_job);
 511         MALI_DEBUG_ASSERT(NULL == group->pp_running_job);
 512
 513         mali_group_lock(group);
 514
 515         group->session = NULL;
 516
 517         if (NULL != group->mmu)
 518         {
 519                 mali_mmu_reset(group->mmu);
 520         }
 521
 522         if (NULL != group->gp_core)
 523         {
 524                 mali_gp_reset(group->gp_core);
 525         }
 526
 527         if (NULL != group->pp_core)
 528         {
 529                 mali_group_reset_pp(group);
 530         }
 531
 532         mali_group_unlock(group);
 533 }
 534
 535 struct mali_gp_core* mali_group_get_gp_core(struct mali_group *group)
 536 {
 537         return group->gp_core;
 538 }
 539
 540 struct mali_pp_core* mali_group_get_pp_core(struct mali_group *group)
 541 {
 542         return group->pp_core;
 543 }
 544
 545 _mali_osk_errcode_t mali_group_start_gp_job(struct mali_group *group, struct mali_gp_job *job)
 546 {
 547         struct mali_session_data *session;
 548         enum mali_group_activate_pd_status activate_status;
 549
 550         MALI_ASSERT_GROUP_LOCKED(group);
 551         MALI_DEBUG_ASSERT(MALI_GROUP_STATE_IDLE == group->state);
 552
 553         session = mali_gp_job_get_session(job);
 554
 555         if (NULL != group->l2_cache_core[0])
 556         {
 557                 mali_l2_cache_invalidate_all_conditional(group->l2_cache_core[0], mali_gp_job_get_id(job));
 558         }
 559
 560         activate_status = mali_group_activate_page_directory(group, session);
 561         if (MALI_GROUP_ACTIVATE_PD_STATUS_FAILED != activate_status)
 562         {
 563                 /* if session is NOT kept Zapping is done as part of session switch */
 564                 if (MALI_GROUP_ACTIVATE_PD_STATUS_OK_KEPT_PD == activate_status)
 565                 {
 566                         mali_mmu_zap_tlb_without_stall(group->mmu);
 567                 }
 568                 mali_gp_job_start(group->gp_core, job);
 569
 570                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 571                                 MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0) |
 572                                 MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH,
 573                         mali_gp_job_get_frame_builder_id(job), mali_gp_job_get_flush_id(job), 0, 0, 0);
 574                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START |
 575                                 MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0),
 576                                 mali_gp_job_get_pid(job), mali_gp_job_get_tid(job), 0, 0, 0);
 577 #if defined(CONFIG_MALI400_PROFILING)
 578                 if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) &&
 579                                 (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0])))
 580                         mali_group_report_l2_cache_counters_per_core(group, 0);
 581 #endif /* #if defined(CONFIG_MALI400_PROFILING) */
 582
 583                 group->gp_running_job = job;
 584                 group->state = MALI_GROUP_STATE_WORKING;
 585
 586                 /* Setup the timeout timer value and save the job id for the job running on the gp core */
 587                 _mali_osk_timer_mod(group->timeout_timer, _mali_osk_time_mstoticks(mali_max_job_runtime));
 588
 589                 return _MALI_OSK_ERR_OK;
 590         }
 591
 592         return _MALI_OSK_ERR_FAULT;
 593 }
 594
 595 _mali_osk_errcode_t mali_group_start_pp_job(struct mali_group *group, struct mali_pp_job *job, u32 sub_job)
 596 {
 597         struct mali_session_data *session;
 598         enum mali_group_activate_pd_status activate_status;
 599
 600         MALI_ASSERT_GROUP_LOCKED(group);
 601         MALI_DEBUG_ASSERT(MALI_GROUP_STATE_IDLE == group->state);
 602
 603         session = mali_pp_job_get_session(job);
 604
 605         if (NULL != group->l2_cache_core[0])
 606         {
 607                 mali_l2_cache_invalidate_all_conditional(group->l2_cache_core[0], mali_pp_job_get_id(job));
 608         }
 609
 610         if (NULL != group->l2_cache_core[1])
 611         {
 612                 mali_l2_cache_invalidate_all_conditional(group->l2_cache_core[1], mali_pp_job_get_id(job));
 613         }
 614
 615         activate_status = mali_group_activate_page_directory(group, session);
 616         if (MALI_GROUP_ACTIVATE_PD_STATUS_FAILED != activate_status)
 617         {
 618                 /* if session is NOT kept Zapping is done as part of session switch */
 619                 if (MALI_GROUP_ACTIVATE_PD_STATUS_OK_KEPT_PD == activate_status)
 620                 {
 621                         MALI_DEBUG_PRINT(3, ("PP starting job PD_Switch 0 Flush 1 Zap 1\n"));
 622                         mali_mmu_zap_tlb_without_stall(group->mmu);
 623                 }
 624
 625                 if (mali_group_is_virtual(group))
 626                 {
 627                         struct mali_group *child;
 628                         struct mali_group *temp;
 629                         u32 core_num = 0;
 630
 631                         /* Configure DLBU for the job */
 632                         mali_dlbu_config_job(group->dlbu_core, job);
 633
 634                         /* Write stack address for each child group */
 635                         _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list)
 636                         {
 637                                 mali_pp_write_addr_stack(child->pp_core, job);
 638                                 core_num++;
 639                         }
 640                 }
 641
 642                 mali_pp_job_start(group->pp_core, job, sub_job, MALI_FALSE);
 643
 644                 /* if the group is virtual, loop through physical groups which belong to this group
 645                  * and call profiling events for its cores as virtual */
 646                 if (MALI_TRUE == mali_group_is_virtual(group))
 647                 {
 648                         struct mali_group *child;
 649                         struct mali_group *temp;
 650
 651                         _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list)
 652                         {
 653                                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE|
 654                                                               MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core))|
 655                                                               MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH,
 656                                                               mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job), 0, 0, 0);
 657
 658                                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START|
 659                                                               MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core))|
 660                                                               MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL,
 661                                                               mali_pp_job_get_pid(job), mali_pp_job_get_tid(job), 0, 0, 0);
 662                         }
 663 #if defined(CONFIG_MALI400_PROFILING)
 664                         if (0 != group->l2_cache_core_ref_count[0])
 665                         {
 666                                 if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) &&
 667                                                                         (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0])))
 668                                 {
 669                                         mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0]));
 670                                 }
 671                         }
 672                         if (0 != group->l2_cache_core_ref_count[1])
 673                         {
 674                                 if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[1])) &&
 675                                                                         (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[1])))
 676                                 {
 677                                         mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[1]));
 678                                 }
 679                         }
 680 #endif /* #if defined(CONFIG_MALI400_PROFILING) */
 681                 }
 682                 else /* group is physical - call profiling events for physical cores */
 683                 {
 684                         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE|
 685                                                       MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(group->pp_core))|
 686                                                       MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH,
 687                                                       mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job), 0, 0, 0);
 688
 689                         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START|
 690                                                       MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(group->pp_core))|
 691                                                       MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL,
 692                                                       mali_pp_job_get_pid(job), mali_pp_job_get_tid(job), 0, 0, 0);
 693 #if defined(CONFIG_MALI400_PROFILING)
 694                         if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) &&
 695                                         (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0])))
 696                         {
 697                                 mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0]));
 698                         }
 699 #endif /* #if defined(CONFIG_MALI400_PROFILING) */
 700                 }
 701                 group->pp_running_job = job;
 702                 group->pp_running_sub_job = sub_job;
 703                 group->state = MALI_GROUP_STATE_WORKING;
 704
 705                 /* Setup the timeout timer value and save the job id for the job running on the pp core */
 706                 _mali_osk_timer_mod(group->timeout_timer, _mali_osk_time_mstoticks(mali_max_job_runtime));
 707
 708                 return _MALI_OSK_ERR_OK;
 709         }
 710
 711         return _MALI_OSK_ERR_FAULT;
 712 }
 713
 714 struct mali_gp_job *mali_group_resume_gp_with_new_heap(struct mali_group *group, u32 job_id, u32 start_addr, u32 end_addr)
 715 {
 716         MALI_ASSERT_GROUP_LOCKED(group);
 717
 718         if (group->state != MALI_GROUP_STATE_OOM ||
 719             mali_gp_job_get_id(group->gp_running_job) != job_id)
 720         {
 721                 return NULL; /* Illegal request or job has already been aborted */
 722         }
 723
 724         if (NULL != group->l2_cache_core[0])
 725         {
 726                 mali_l2_cache_invalidate_all_force(group->l2_cache_core[0]);
 727         }
 728
 729         mali_mmu_zap_tlb_without_stall(group->mmu);
 730
 731         mali_gp_resume_with_new_heap(group->gp_core, start_addr, end_addr);
 732
 733         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_RESUME|MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0), 0, 0, 0, 0, 0);
 734
 735         group->state = MALI_GROUP_STATE_WORKING;
 736
 737         return group->gp_running_job;
 738 }
 739
 740 static void mali_group_reset_pp(struct mali_group *group)
 741 {
 742         struct mali_group *child;
 743         struct mali_group *temp;
 744
 745         /* TODO: If we *know* that the group is idle, this could be faster. */
 746
 747         mali_pp_reset_async(group->pp_core);
 748
 749 //      if (!mali_group_is_virtual(group) || NULL == group->pp_running_job) //LWJ
 750 /* MALI_SEC */
 751         if (!mali_group_is_virtual(group) || NULL != group->pp_running_job)
 752         {
 753                 /* This is a physical group or an idle virtual group -- simply wait for
 754                  * the reset to complete. */
 755                 mali_pp_reset_wait(group->pp_core);
 756         }
 757         else /* virtual group */
 758         {
 759                 /* Loop through all members of this virtual group and wait until they
 760                  * are done resetting.
 761                  */
 762                 _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list)
 763                 {
 764                         mali_pp_reset_wait(child->pp_core);
 765                 }
 766         }
 767 }
 768
 769 static void mali_group_complete_pp(struct mali_group *group, mali_bool success)
 770 {
 771         struct mali_pp_job *pp_job_to_return;
 772         u32 pp_sub_job_to_return;
 773
 774         MALI_DEBUG_ASSERT_POINTER(group->pp_core);
 775         MALI_DEBUG_ASSERT_POINTER(group->pp_running_job);
 776         MALI_ASSERT_GROUP_LOCKED(group);
 777
 778         mali_group_post_process_job_pp(group);
 779
 780         mali_pp_reset_async(group->pp_core);
 781
 782         pp_job_to_return = group->pp_running_job;
 783         pp_sub_job_to_return = group->pp_running_sub_job;
 784         group->state = MALI_GROUP_STATE_IDLE;
 785         group->pp_running_job = NULL;
 786
 787         mali_group_deactivate_page_directory(group, group->session);
 788
 789         if (_MALI_OSK_ERR_OK != mali_pp_reset_wait(group->pp_core))
 790         {
 791                 MALI_DEBUG_PRINT(3, ("Mali group: Failed to reset PP, need to reset entire group\n"));
 792
 793                 mali_group_recovery_reset(group);
 794         }
 795
 796         mali_pp_scheduler_job_done(group, pp_job_to_return, pp_sub_job_to_return, success);
 797 }
 798
 799 static void mali_group_complete_gp(struct mali_group *group, mali_bool success)
 800 {
 801         struct mali_gp_job *gp_job_to_return;
 802
 803         MALI_DEBUG_ASSERT_POINTER(group->gp_core);
 804         MALI_DEBUG_ASSERT_POINTER(group->gp_running_job);
 805         MALI_ASSERT_GROUP_LOCKED(group);
 806
 807         mali_group_post_process_job_gp(group, MALI_FALSE);
 808
 809         mali_gp_reset_async(group->gp_core);
 810
 811         gp_job_to_return = group->gp_running_job;
 812         group->state = MALI_GROUP_STATE_IDLE;
 813         group->gp_running_job = NULL;
 814
 815         mali_group_deactivate_page_directory(group, group->session);
 816
 817         if (_MALI_OSK_ERR_OK != mali_gp_reset_wait(group->gp_core))
 818         {
 819                 MALI_DEBUG_PRINT(3, ("Mali group: Failed to reset GP, need to reset entire group\n"));
 820
 821                 mali_group_recovery_reset(group);
 822         }
 823
 824         mali_gp_scheduler_job_done(group, gp_job_to_return, success);
 825 }
 826
 827 void mali_group_abort_gp_job(struct mali_group *group, u32 job_id)
 828 {
 829         MALI_ASSERT_GROUP_LOCKED(group);
 830
 831         if (group->state == MALI_GROUP_STATE_IDLE ||
 832             mali_gp_job_get_id(group->gp_running_job) != job_id)
 833         {
 834                 return; /* No need to cancel or job has already been aborted or completed */
 835         }
 836
 837         mali_group_complete_gp(group, MALI_FALSE);
 838 }
 839
 840 static void mali_group_abort_pp_job(struct mali_group *group, u32 job_id)
 841 {
 842         MALI_ASSERT_GROUP_LOCKED(group);
 843
 844         if (group->state == MALI_GROUP_STATE_IDLE ||
 845             mali_pp_job_get_id(group->pp_running_job) != job_id)
 846         {
 847                 return; /* No need to cancel or job has already been aborted or completed */
 848         }
 849
 850         mali_group_complete_pp(group, MALI_FALSE);
 851 }
 852
 853 void mali_group_abort_session(struct mali_group *group, struct mali_session_data *session)
 854 {
 855         struct mali_gp_job *gp_job;
 856         struct mali_pp_job *pp_job;
 857         u32 gp_job_id = 0;
 858         u32 pp_job_id = 0;
 859         mali_bool abort_pp = MALI_FALSE;
 860         mali_bool abort_gp = MALI_FALSE;
 861
 862         mali_group_lock(group);
 863
 864         if (mali_group_is_in_virtual(group))
 865         {
 866                 /* Group is member of a virtual group, don't touch it! */
 867                 mali_group_unlock(group);
 868                 return;
 869         }
 870
 871         gp_job = group->gp_running_job;
 872         pp_job = group->pp_running_job;
 873
 874         if ((NULL != gp_job) && (mali_gp_job_get_session(gp_job) == session))
 875         {
 876                 MALI_DEBUG_PRINT(4, ("Aborting GP job 0x%08x from session 0x%08x\n", gp_job, session));
 877
 878                 gp_job_id = mali_gp_job_get_id(gp_job);
 879                 abort_gp = MALI_TRUE;
 880         }
 881
 882         if ((NULL != pp_job) && (mali_pp_job_get_session(pp_job) == session))
 883         {
 884                 MALI_DEBUG_PRINT(4, ("Mali group: Aborting PP job 0x%08x from session 0x%08x\n", pp_job, session));
 885
 886                 pp_job_id = mali_pp_job_get_id(pp_job);
 887                 abort_pp = MALI_TRUE;
 888         }
 889
 890 //      if (0 != abort_gp) //LWJ ORG
 891         if (abort_gp)
 892         {
 893                 mali_group_abort_gp_job(group, gp_job_id);
 894         }
 895 //      if (0 != abort_pp) //LWJ ORG
 896         if (abort_pp)
 897         {
 898                 mali_group_abort_pp_job(group, pp_job_id);
 899         }
 900
 901         mali_group_remove_session_if_unused(group, session);
 902
 903         mali_group_unlock(group);
 904 }
 905
 906 struct mali_group *mali_group_get_glob_group(u32 index)
 907 {
 908         if(mali_global_num_groups > index)
 909         {
 910                 return mali_global_groups[index];
 911         }
 912
 913         return NULL;
 914 }
 915
 916 u32 mali_group_get_glob_num_groups(void)
 917 {
 918         return mali_global_num_groups;
 919 }
 920
 921 static enum mali_group_activate_pd_status mali_group_activate_page_directory(struct mali_group *group, struct mali_session_data *session)
 922 {
 923         enum mali_group_activate_pd_status retval;
 924         MALI_ASSERT_GROUP_LOCKED(group);
 925
 926         MALI_DEBUG_PRINT(5, ("Mali group: Activating page directory 0x%08X from session 0x%08X on group 0x%08X\n", mali_session_get_page_directory(session), session, group));
 927         MALI_DEBUG_ASSERT(0 <= group->page_dir_ref_count);
 928
 929         if (0 != group->page_dir_ref_count)
 930         {
 931                 if (group->session != session)
 932                 {
 933                         MALI_DEBUG_PRINT(4, ("Mali group: Activating session FAILED: 0x%08x on group 0x%08X. Existing session: 0x%08x\n", session, group, group->session));
 934                         return MALI_GROUP_ACTIVATE_PD_STATUS_FAILED;
 935                 }
 936                 else
 937                 {
 938                         MALI_DEBUG_PRINT(4, ("Mali group: Activating session already activated: 0x%08x on group 0x%08X. New Ref: %d\n", session, group, 1+group->page_dir_ref_count));
 939                         retval = MALI_GROUP_ACTIVATE_PD_STATUS_OK_KEPT_PD;
 940
 941                 }
 942         }
 943         else
 944         {
 945                 /* There might be another session here, but it is ok to overwrite it since group->page_dir_ref_count==0 */
 946                 if (group->session != session)
 947                 {
 948                         mali_bool activate_success;
 949                         MALI_DEBUG_PRINT(5, ("Mali group: Activate session: %08x previous: %08x on group 0x%08X. Ref: %d\n", session, group->session, group, 1+group->page_dir_ref_count));
 950
 951                         activate_success = mali_mmu_activate_page_directory(group->mmu, mali_session_get_page_directory(session));
 952                         MALI_DEBUG_ASSERT(activate_success);
 953                         if ( MALI_FALSE== activate_success ) return MALI_GROUP_ACTIVATE_PD_STATUS_FAILED;
 954                         group->session = session;
 955                         retval = MALI_GROUP_ACTIVATE_PD_STATUS_OK_SWITCHED_PD;
 956                 }
 957                 else
 958                 {
 959                         MALI_DEBUG_PRINT(4, ("Mali group: Activate existing session 0x%08X on group 0x%08X. Ref: %d\n", session->page_directory, group, 1+group->page_dir_ref_count));
 960                         retval = MALI_GROUP_ACTIVATE_PD_STATUS_OK_KEPT_PD;
 961                 }
 962         }
 963
 964         group->page_dir_ref_count++;
 965         return retval;
 966 }
 967
 968 static void mali_group_deactivate_page_directory(struct mali_group *group, struct mali_session_data *session)
 969 {
 970         MALI_ASSERT_GROUP_LOCKED(group);
 971
 972         MALI_DEBUG_ASSERT(0 < group->page_dir_ref_count);
 973         MALI_DEBUG_ASSERT(session == group->session);
 974
 975         group->page_dir_ref_count--;
 976
 977         /* As an optimization, the MMU still points to the group->session even if (0 == group->page_dir_ref_count),
 978            and we do not call mali_mmu_activate_empty_page_directory(group->mmu); */
 979         MALI_DEBUG_ASSERT(0 <= group->page_dir_ref_count);
 980 }
 981
 982 static void mali_group_remove_session_if_unused(struct mali_group *group, struct mali_session_data *session)
 983 {
 984         MALI_ASSERT_GROUP_LOCKED(group);
 985
 986         if (0 == group->page_dir_ref_count)
 987         {
 988                 MALI_DEBUG_ASSERT(MALI_GROUP_STATE_WORKING != group->state);
 989
 990                 if (group->session == session)
 991                 {
 992                         MALI_DEBUG_ASSERT(MALI_TRUE == group->power_is_on);
 993                         MALI_DEBUG_PRINT(3, ("Mali group: Deactivating unused session 0x%08X on group %08X\n", session, group));
 994                         mali_mmu_activate_empty_page_directory(group->mmu);
 995                         group->session = NULL;
 996                 }
 997         }
 998 }
 999
1000 void mali_group_power_on(void)
1001 {
1002         int i;
1003         for (i = 0; i < mali_global_num_groups; i++)
1004         {
1005                 struct mali_group *group = mali_global_groups[i];
1006                 mali_group_lock(group);
1007                 MALI_DEBUG_ASSERT(MALI_GROUP_STATE_IDLE == group->state);
1008                 group->power_is_on = MALI_TRUE;
1009
1010                 if (NULL != group->l2_cache_core[0])
1011                 {
1012                         mali_l2_cache_power_is_enabled_set(group->l2_cache_core[0], MALI_TRUE);
1013                 }
1014
1015                 if (NULL != group->l2_cache_core[1])
1016                 {
1017                         mali_l2_cache_power_is_enabled_set(group->l2_cache_core[1], MALI_TRUE);
1018                 }
1019
1020                 mali_group_unlock(group);
1021         }
1022         MALI_DEBUG_PRINT(4,("group: POWER ON\n"));
1023 }
1024
1025 mali_bool mali_group_power_is_on(struct mali_group *group)
1026 {
1027         MALI_ASSERT_GROUP_LOCKED(group);
1028         return group->power_is_on;
1029 }
1030
1031 void mali_group_power_off(void)
1032 {
1033         int i;
1034         /* It is necessary to set group->session = NULL; so that the powered off MMU is not written to on map /unmap */
1035         /* It is necessary to set group->power_is_on=MALI_FALSE so that pending bottom_halves does not access powered off cores. */
1036         for (i = 0; i < mali_global_num_groups; i++)
1037         {
1038                 struct mali_group *group = mali_global_groups[i];
1039                 mali_group_lock(group);
1040                 MALI_DEBUG_ASSERT(MALI_GROUP_STATE_IDLE == group->state);
1041                 group->session = NULL;
1042                 group->power_is_on = MALI_FALSE;
1043
1044                 if (NULL != group->l2_cache_core[0])
1045                 {
1046                         mali_l2_cache_power_is_enabled_set(group->l2_cache_core[0], MALI_FALSE);
1047                 }
1048
1049                 if (NULL != group->l2_cache_core[1])
1050                 {
1051                         mali_l2_cache_power_is_enabled_set(group->l2_cache_core[1], MALI_FALSE);
1052                 }
1053
1054                 mali_group_unlock(group);
1055         }
1056         MALI_DEBUG_PRINT(4,("group: POWER OFF\n"));
1057 }
1058
1059
1060 static void mali_group_recovery_reset(struct mali_group *group)
1061 {
1062         MALI_ASSERT_GROUP_LOCKED(group);
1063
1064         /* Stop cores, bus stop */
1065         if (NULL != group->pp_core)
1066         {
1067                 mali_pp_stop_bus(group->pp_core);
1068         }
1069         else
1070         {
1071                 mali_gp_stop_bus(group->gp_core);
1072         }
1073
1074         /* Flush MMU and clear page fault (if any) */
1075         mali_mmu_activate_fault_flush_page_directory(group->mmu);
1076         mali_mmu_page_fault_done(group->mmu);
1077
1078         /* Wait for cores to stop bus, then do a hard reset on them */
1079         if (NULL != group->pp_core)
1080         {
1081                 if (mali_group_is_virtual(group))
1082                 {
1083                         struct mali_group *child, *temp;
1084
1085                         _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list)
1086                         {
1087                                 mali_pp_stop_bus_wait(child->pp_core);
1088                                 mali_pp_hard_reset(child->pp_core);
1089                         }
1090                 }
1091                 else
1092                 {
1093                         mali_pp_stop_bus_wait(group->pp_core);
1094                         mali_pp_hard_reset(group->pp_core);
1095                 }
1096         }
1097         else
1098         {
1099                 mali_gp_stop_bus_wait(group->gp_core);
1100                 mali_gp_hard_reset(group->gp_core);
1101         }
1102
1103         /* Reset MMU */
1104         mali_mmu_reset(group->mmu);
1105         group->session = NULL;
1106 }
1107
1108 #if MALI_STATE_TRACKING
1109 u32 mali_group_dump_state(struct mali_group *group, char *buf, u32 size)
1110 {
1111         int n = 0;
1112
1113         n += _mali_osk_snprintf(buf + n, size - n, "Group: %p\n", group);
1114         n += _mali_osk_snprintf(buf + n, size - n, "\tstate: %d\n", group->state);
1115         if (group->gp_core)
1116         {
1117                 n += mali_gp_dump_state(group->gp_core, buf + n, size - n);
1118                 n += _mali_osk_snprintf(buf + n, size - n, "\tGP job: %p\n", group->gp_running_job);
1119         }
1120         if (group->pp_core)
1121         {
1122                 n += mali_pp_dump_state(group->pp_core, buf + n, size - n);
1123                 n += _mali_osk_snprintf(buf + n, size - n, "\tPP job: %p, subjob %d \n",
1124                                         group->pp_running_job, group->pp_running_sub_job);
1125         }
1126
1127         return n;
1128 }
1129 #endif
1130
1131 static void mali_group_mmu_page_fault(struct mali_group *group)
1132 {
1133         MALI_ASSERT_GROUP_LOCKED(group);
1134
1135         if (NULL != group->pp_core)
1136         {
1137                 struct mali_pp_job *pp_job_to_return;
1138                 u32 pp_sub_job_to_return;
1139
1140                 MALI_DEBUG_ASSERT_POINTER(group->pp_running_job);
1141
1142                 mali_group_post_process_job_pp(group);
1143
1144                 pp_job_to_return = group->pp_running_job;
1145                 pp_sub_job_to_return = group->pp_running_sub_job;
1146                 group->state = MALI_GROUP_STATE_IDLE;
1147                 group->pp_running_job = NULL;
1148
1149                 mali_group_deactivate_page_directory(group, group->session);
1150
1151                 mali_group_recovery_reset(group); /* This will also clear the page fault itself */
1152
1153                 mali_pp_scheduler_job_done(group, pp_job_to_return, pp_sub_job_to_return, MALI_FALSE);
1154         }
1155         else
1156         {
1157                 struct mali_gp_job *gp_job_to_return;
1158
1159                 MALI_DEBUG_ASSERT_POINTER(group->gp_running_job);
1160
1161                 mali_group_post_process_job_gp(group, MALI_FALSE);
1162
1163                 gp_job_to_return = group->gp_running_job;
1164                 group->state = MALI_GROUP_STATE_IDLE;
1165                 group->gp_running_job = NULL;
1166
1167                 mali_group_deactivate_page_directory(group, group->session);
1168
1169                 mali_group_recovery_reset(group); /* This will also clear the page fault itself */
1170
1171                 mali_gp_scheduler_job_done(group, gp_job_to_return, MALI_FALSE);
1172         }
1173 }
1174
1175 _mali_osk_errcode_t mali_group_upper_half_mmu(void * data)
1176 {
1177         struct mali_group *group = (struct mali_group *)data;
1178         struct mali_mmu_core *mmu = group->mmu;
1179         u32 int_stat;
1180
1181         MALI_DEBUG_ASSERT_POINTER(mmu);
1182
1183         /* Check if it was our device which caused the interrupt (we could be sharing the IRQ line) */
1184         int_stat = mali_mmu_get_int_status(mmu);
1185         if (0 != int_stat)
1186         {
1187                 struct mali_group *parent = group->parent_group;
1188
1189                 /* page fault or bus error, we thread them both in the same way */
1190                 mali_mmu_mask_all_interrupts(mmu);
1191                 if (NULL == parent)
1192                 {
1193                         _mali_osk_wq_schedule_work(group->bottom_half_work_mmu);
1194                 }
1195                 else
1196                 {
1197                         _mali_osk_wq_schedule_work(parent->bottom_half_work_mmu);
1198                 }
1199                 return _MALI_OSK_ERR_OK;
1200         }
1201
1202         return _MALI_OSK_ERR_FAULT;
1203 }
1204
1205 static void mali_group_bottom_half_mmu(void * data)
1206 {
1207         struct mali_group *group = (struct mali_group *)data;
1208         struct mali_mmu_core *mmu = group->mmu;
1209         u32 rawstat;
1210         u32 status;
1211
1212         MALI_DEBUG_ASSERT_POINTER(mmu);
1213
1214         mali_group_lock(group);
1215
1216         /* TODO: Remove some of these asserts? Will we ever end up in
1217          * "physical" bottom half for a member of the virtual group? */
1218         MALI_DEBUG_ASSERT(NULL == group->parent_group);
1219         MALI_DEBUG_ASSERT(!mali_group_is_in_virtual(group));
1220
1221         if ( MALI_FALSE == mali_group_power_is_on(group) )
1222         {
1223                 MALI_PRINT_ERROR(("Interrupt bottom half of %s when core is OFF.", mmu->hw_core.description));
1224                 mali_group_unlock(group);
1225                 return;
1226         }
1227
1228         rawstat = mali_mmu_get_rawstat(mmu);
1229         status = mali_mmu_get_status(mmu);
1230
1231         MALI_DEBUG_PRINT(4, ("Mali MMU: Bottom half, interrupt 0x%08X, status 0x%08X\n", rawstat, status));
1232
1233         if (rawstat & (MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR))
1234         {
1235                 /* An actual page fault has occurred. */
1236                 u32 fault_address = mali_mmu_get_page_fault_addr(mmu);
1237                 MALI_DEBUG_PRINT(2,("Mali MMU: Page fault detected at 0x%x from bus id %d of type %s on %s\n",
1238                                  (void*)fault_address,
1239                                  (status >> 6) & 0x1F,
1240                                  (status & 32) ? "write" : "read",
1241                                  mmu->hw_core.description));
1242                 MALI_IGNORE(fault_address);
1243
1244                 mali_group_mmu_page_fault(group);
1245         }
1246
1247         mali_group_unlock(group);
1248 }
1249
1250 _mali_osk_errcode_t mali_group_upper_half_gp(void *data)
1251 {
1252         struct mali_group *group = (struct mali_group *)data;
1253         struct mali_gp_core *core = group->gp_core;
1254         u32 irq_readout;
1255
1256         irq_readout = mali_gp_get_int_stat(core);
1257
1258         if (MALIGP2_REG_VAL_IRQ_MASK_NONE != irq_readout)
1259         {
1260                 /* Mask out all IRQs from this core until IRQ is handled */
1261                 mali_gp_mask_all_interrupts(core);
1262
1263                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE|MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0)|MALI_PROFILING_EVENT_REASON_SINGLE_HW_INTERRUPT, irq_readout, 0, 0, 0, 0);
1264
1265                 /* We do need to handle this in a bottom half */
1266                 _mali_osk_wq_schedule_work(group->bottom_half_work_gp);
1267                 return _MALI_OSK_ERR_OK;
1268         }
1269
1270         return _MALI_OSK_ERR_FAULT;
1271 }
1272
1273 static void mali_group_bottom_half_gp(void *data)
1274 {
1275         struct mali_group *group = (struct mali_group *)data;
1276         u32 irq_readout;
1277         u32 irq_errors;
1278
1279         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START|MALI_PROFILING_EVENT_CHANNEL_SOFTWARE|MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, 0, _mali_osk_get_tid(), MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), 0, 0);
1280
1281         mali_group_lock(group);
1282
1283         if ( MALI_FALSE == mali_group_power_is_on(group) )
1284         {
1285                 MALI_PRINT_ERROR(("Mali group: Interrupt bottom half of %s when core is OFF.", mali_gp_get_hw_core_desc(group->gp_core)));
1286                 mali_group_unlock(group);
1287                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP|MALI_PROFILING_EVENT_CHANNEL_SOFTWARE|MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, 0, _mali_osk_get_tid(), 0, 0, 0);
1288                 return;
1289         }
1290
1291         irq_readout = mali_gp_read_rawstat(group->gp_core);
1292
1293         MALI_DEBUG_PRINT(4, ("Mali group: GP bottom half IRQ 0x%08X from core %s\n", irq_readout, mali_gp_get_hw_core_desc(group->gp_core)));
1294
1295         if (irq_readout & (MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST|MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST))
1296         {
1297                 u32 core_status = mali_gp_read_core_status(group->gp_core);
1298                 if (0 == (core_status & MALIGP2_REG_VAL_STATUS_MASK_ACTIVE))
1299                 {
1300                         MALI_DEBUG_PRINT(4, ("Mali group: GP job completed, calling group handler\n"));
1301                         group->core_timed_out = MALI_FALSE;
1302                         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1303                                                       MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1304                                                       MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1305                                                       0, _mali_osk_get_tid(), 0, 0, 0);
1306                         mali_group_complete_gp(group, MALI_TRUE);
1307                         mali_group_unlock(group);
1308                         return;
1309                 }
1310         }
1311
1312         /*
1313          * Now lets look at the possible error cases (IRQ indicating error or timeout)
1314          * END_CMD_LST, HANG and PLBU_OOM interrupts are not considered error.
1315          */
1316         irq_errors = irq_readout & ~(MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST|MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST|MALIGP2_REG_VAL_IRQ_HANG|MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM);
1317         if (0 != irq_errors)
1318         {
1319                 MALI_PRINT_ERROR(("Mali group: Unknown interrupt 0x%08X from core %s, aborting job\n", irq_readout, mali_gp_get_hw_core_desc(group->gp_core)));
1320                 group->core_timed_out = MALI_FALSE;
1321                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1322                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1323                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1324                                               0, _mali_osk_get_tid(), 0, 0, 0);
1325                 mali_group_complete_gp(group, MALI_FALSE);
1326                 mali_group_unlock(group);
1327                 return;
1328         }
1329         else if (group->core_timed_out) /* SW timeout */
1330         {
1331                 group->core_timed_out = MALI_FALSE;
1332                 if (!_mali_osk_timer_pending(group->timeout_timer) && NULL != group->gp_running_job)
1333                 {
1334                         MALI_PRINT(("Mali group: Job %d timed out\n", mali_gp_job_get_id(group->gp_running_job)));
1335                         mali_group_complete_gp(group, MALI_FALSE);
1336                         mali_group_unlock(group);
1337                         return;
1338                 }
1339         }
1340         else if (irq_readout & MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM)
1341         {
1342                 /* GP wants more memory in order to continue. */
1343                 MALI_DEBUG_PRINT(3, ("Mali group: PLBU needs more heap memory\n"));
1344
1345                 group->state = MALI_GROUP_STATE_OOM;
1346                 mali_group_unlock(group); /* Nothing to do on the HW side, so just release group lock right away */
1347                 mali_gp_scheduler_oom(group, group->gp_running_job);
1348                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP|MALI_PROFILING_EVENT_CHANNEL_SOFTWARE|MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, 0, _mali_osk_get_tid(), 0, 0, 0);
1349                 return;
1350         }
1351
1352         /*
1353          * The only way to get here is if we only got one of two needed END_CMD_LST
1354          * interrupts. Enable all but not the complete interrupt that has been
1355          * received and continue to run.
1356          */
1357         mali_gp_enable_interrupts(group->gp_core, irq_readout & (MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST|MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST));
1358         mali_group_unlock(group);
1359
1360         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP|MALI_PROFILING_EVENT_CHANNEL_SOFTWARE|MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, 0, _mali_osk_get_tid(), 0, 0, 0);
1361 }
1362
1363 static void mali_group_post_process_job_gp(struct mali_group *group, mali_bool suspend)
1364 {
1365         /* Stop the timeout timer. */
1366         _mali_osk_timer_del_async(group->timeout_timer);
1367
1368         if (NULL == group->gp_running_job)
1369         {
1370                 /* Nothing to do */
1371                 return;
1372         }
1373
1374         mali_gp_update_performance_counters(group->gp_core, group->gp_running_job, suspend);
1375
1376 #if defined(CONFIG_MALI400_PROFILING)
1377         if (suspend)
1378         {
1379                 /* @@@@ todo: test this case and see if it is still working*/
1380                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SUSPEND|MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0),
1381                                               mali_gp_job_get_perf_counter_value0(group->gp_running_job),
1382                                               mali_gp_job_get_perf_counter_value1(group->gp_running_job),
1383                                               mali_gp_job_get_perf_counter_src0(group->gp_running_job) | (mali_gp_job_get_perf_counter_src1(group->gp_running_job) << 8),
1384                                               0, 0);
1385         }
1386         else
1387         {
1388                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP|MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0),
1389                                               mali_gp_job_get_perf_counter_value0(group->gp_running_job),
1390                                               mali_gp_job_get_perf_counter_value1(group->gp_running_job),
1391                                               mali_gp_job_get_perf_counter_src0(group->gp_running_job) | (mali_gp_job_get_perf_counter_src1(group->gp_running_job) << 8),
1392                                               0, 0);
1393
1394                 if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) &&
1395                                 (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0])))
1396                         mali_group_report_l2_cache_counters_per_core(group, 0);
1397         }
1398 #endif
1399
1400         mali_gp_job_set_current_heap_addr(group->gp_running_job,
1401                                           mali_gp_read_plbu_alloc_start_addr(group->gp_core));
1402 }
1403
1404 _mali_osk_errcode_t mali_group_upper_half_pp(void *data)
1405 {
1406         struct mali_group *group = (struct mali_group *)data;
1407         struct mali_pp_core *core = group->pp_core;
1408         u32 irq_readout;
1409
1410         /*
1411          * For Mali-450 there is one particular case we need to watch out for:
1412          *
1413          * Criteria 1) this function call can be due to a shared interrupt,
1414          * and not necessary because this core signaled an interrupt.
1415          * Criteria 2) this core is a part of a virtual group, and thus it should
1416          * not do any post processing.
1417          * Criteria 3) this core has actually indicated that is has completed by
1418          * having set raw_stat/int_stat registers to != 0
1419          *
1420          * If all this criteria is meet, then we could incorrectly start post
1421          * processing on the wrong group object (this should only happen on the
1422          * parent group)
1423          */
1424 #if !defined(MALI_UPPER_HALF_SCHEDULING)
1425         if (mali_group_is_in_virtual(group))
1426         {
1427                 /*
1428                  * This check is done without the group lock held, which could lead to
1429                  * a potential race. This is however ok, since we will safely re-check
1430                  * this with the group lock held at a later stage. This is just an
1431                  * early out which will strongly benefit shared IRQ systems.
1432                  */
1433                 return _MALI_OSK_ERR_OK;
1434         }
1435 #endif
1436
1437         irq_readout = mali_pp_get_int_stat(core);
1438         if (MALI200_REG_VAL_IRQ_MASK_NONE != irq_readout)
1439         {
1440                 /* Mask out all IRQs from this core until IRQ is handled */
1441                 mali_pp_mask_all_interrupts(core);
1442
1443 #if defined(CONFIG_MALI400_PROFILING)
1444                 /* Currently no support for this interrupt event for the virtual PP core */
1445                 if (!mali_group_is_virtual(group))
1446                 {
1447                         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
1448                                                       MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(core->core_id) |
1449                                                       MALI_PROFILING_EVENT_REASON_SINGLE_HW_INTERRUPT,
1450                                                       irq_readout, 0, 0, 0, 0);
1451                 }
1452 #endif
1453
1454 #if defined(MALI_UPPER_HALF_SCHEDULING)
1455                 if (irq_readout & MALI200_REG_VAL_IRQ_END_OF_FRAME)
1456                 {
1457                         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START |
1458                                                       MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1459                                                       MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF,
1460                                                       0, 0, MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP(core->core_id), 0, 0);
1461
1462                         MALI_DEBUG_PRINT(3, ("Mali PP: Job completed, calling group handler from upper half\n"));
1463
1464                         mali_group_lock(group);
1465
1466                         /* Read int stat again */
1467                         irq_readout = mali_pp_read_rawstat(core);
1468                         if (!(irq_readout & MALI200_REG_VAL_IRQ_END_OF_FRAME))
1469                         {
1470                                 /* There was nothing to do */
1471                                 mali_pp_enable_interrupts(core);
1472                                 mali_group_unlock(group);
1473                                 return _MALI_OSK_ERR_OK;
1474                         }
1475
1476                         if (mali_group_is_in_virtual(group))
1477                         {
1478                                 /* We're member of a virtual group, so interrupt should be handled by the virtual group */
1479                                 mali_pp_enable_interrupts(core);
1480                                 mali_group_unlock(group);
1481                                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1482                                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1483                                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF,
1484                                                               0, 0, MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP(core->core_id), 0, 0);
1485                                 return _MALI_OSK_ERR_FAULT;
1486                         }
1487
1488                         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1489                                                       MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1490                                                       MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF,
1491                                                       0, 0, MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP(core->core_id), 0, 0);
1492
1493                         mali_group_complete_pp(group, MALI_TRUE);
1494                         /* No need to enable interrupts again, since the core will be reset while completing the job */
1495
1496                         mali_group_unlock(group);
1497
1498                         return _MALI_OSK_ERR_OK;
1499                 }
1500 #endif
1501
1502                 /* We do need to handle this in a bottom half */
1503                 _mali_osk_wq_schedule_work(group->bottom_half_work_pp);
1504                 return _MALI_OSK_ERR_OK;
1505         }
1506
1507         return _MALI_OSK_ERR_FAULT;
1508 }
1509
1510 static void mali_group_bottom_half_pp(void *data)
1511 {
1512         struct mali_group *group = (struct mali_group *)data;
1513         struct mali_pp_core *core = group->pp_core;
1514         u32 irq_readout;
1515         u32 irq_errors;
1516
1517         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START |
1518                                       MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1519                                       MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1520                                       0, _mali_osk_get_tid(), MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP(core->core_id), 0, 0);
1521
1522         mali_group_lock(group);
1523
1524         if (mali_group_is_in_virtual(group))
1525         {
1526                 /* We're member of a virtual group, so interrupt should be handled by the virtual group */
1527                 mali_pp_enable_interrupts(core);
1528                 mali_group_unlock(group);
1529                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1530                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1531                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1532                                               0, _mali_osk_get_tid(), 0, 0, 0);
1533                 return;
1534         }
1535
1536         if ( MALI_FALSE == mali_group_power_is_on(group) )
1537         {
1538                 MALI_PRINT_ERROR(("Interrupt bottom half of %s when core is OFF.", mali_pp_get_hw_core_desc(core)));
1539                 mali_group_unlock(group);
1540                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1541                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1542                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1543                                               0, _mali_osk_get_tid(), 0, 0, 0);
1544                 return;
1545         }
1546
1547         irq_readout = mali_pp_read_rawstat(group->pp_core);
1548
1549         MALI_DEBUG_PRINT(4, ("Mali PP: Bottom half IRQ 0x%08X from core %s\n", irq_readout, mali_pp_get_hw_core_desc(group->pp_core)));
1550
1551         if (irq_readout & MALI200_REG_VAL_IRQ_END_OF_FRAME)
1552         {
1553                 MALI_DEBUG_PRINT(3, ("Mali PP: Job completed, calling group handler\n"));
1554                 group->core_timed_out = MALI_FALSE;
1555                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1556                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1557                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1558                                               0, _mali_osk_get_tid(), 0, 0, 0);
1559                 mali_group_complete_pp(group, MALI_TRUE);
1560                 mali_group_unlock(group);
1561                 return;
1562         }
1563
1564         /*
1565          * Now lets look at the possible error cases (IRQ indicating error or timeout)
1566          * END_OF_FRAME and HANG interrupts are not considered error.
1567          */
1568         irq_errors = irq_readout & ~(MALI200_REG_VAL_IRQ_END_OF_FRAME|MALI200_REG_VAL_IRQ_HANG);
1569         if (0 != irq_errors)
1570         {
1571                 MALI_PRINT_ERROR(("Mali PP: Unknown interrupt 0x%08X from core %s, aborting job\n",
1572                                   irq_readout, mali_pp_get_hw_core_desc(group->pp_core)));
1573                 group->core_timed_out = MALI_FALSE;
1574                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1575                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1576                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1577                                               0, _mali_osk_get_tid(), 0, 0, 0);
1578                 mali_group_complete_pp(group, MALI_FALSE);
1579                 mali_group_unlock(group);
1580                 return;
1581         }
1582         else if (group->core_timed_out) /* SW timeout */
1583         {
1584                 group->core_timed_out = MALI_FALSE;
1585                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1586                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1587                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1588                                               0, _mali_osk_get_tid(), 0, 0, 0);
1589                 if (!_mali_osk_timer_pending(group->timeout_timer) && NULL != group->pp_running_job)
1590                 {
1591                         MALI_PRINT(("Mali PP: Job %d timed out on core %s\n",
1592                                     mali_pp_job_get_id(group->pp_running_job), mali_pp_get_hw_core_desc(core)));
1593                         mali_group_complete_pp(group, MALI_FALSE);
1594                         mali_group_unlock(group);
1595                 }
1596                 else
1597                 {
1598                         mali_group_unlock(group);
1599                 }
1600                 return;
1601         }
1602
1603         /*
1604          * We should never get here, re-enable interrupts and continue
1605          */
1606         if (0 == irq_readout)
1607         {
1608                 MALI_DEBUG_PRINT(3, ("Mali group: No interrupt found on core %s\n",
1609                                     mali_pp_get_hw_core_desc(group->pp_core)));
1610         }
1611         else
1612         {
1613                 MALI_PRINT_ERROR(("Mali group: Unhandled PP interrupt 0x%08X on %s\n", irq_readout,
1614                                     mali_pp_get_hw_core_desc(group->pp_core)));
1615         }
1616         mali_pp_enable_interrupts(core);
1617         mali_group_unlock(group);
1618
1619         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1620                                       MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1621                                       MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1622                                       0, _mali_osk_get_tid(), 0, 0, 0);
1623 }
1624
1625 static void mali_group_post_process_job_pp(struct mali_group *group)
1626 {
1627         MALI_ASSERT_GROUP_LOCKED(group);
1628
1629         /* Stop the timeout timer. */
1630         _mali_osk_timer_del_async(group->timeout_timer);
1631
1632         /*todo add stop SW counters profiling*/
1633
1634         if (NULL != group->pp_running_job)
1635         {
1636                 if (MALI_TRUE == mali_group_is_virtual(group))
1637                 {
1638                         struct mali_group *child;
1639                         struct mali_group *temp;
1640
1641                         /* update performance counters from each physical pp core within this virtual group */
1642                         _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list)
1643                         {
1644                                 mali_pp_update_performance_counters(child->pp_core, group->pp_running_job, group->pp_running_sub_job);
1645                         }
1646
1647 #if defined(CONFIG_MALI400_PROFILING)
1648                         /* send profiling data per physical core */
1649                         _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list)
1650                         {
1651                                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP|
1652                                                               MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core))|
1653                                                               MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL,
1654                                                               mali_pp_job_get_perf_counter_value0(group->pp_running_job, mali_pp_core_get_id(child->pp_core)),
1655                                                               mali_pp_job_get_perf_counter_value1(group->pp_running_job, mali_pp_core_get_id(child->pp_core)),
1656                                                               mali_pp_job_get_perf_counter_src0(group->pp_running_job) | (mali_pp_job_get_perf_counter_src1(group->pp_running_job) << 8),
1657                                                               0, 0);
1658                         }
1659                         if (0 != group->l2_cache_core_ref_count[0])
1660                         {
1661                                 if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) &&
1662                                                                         (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0])))
1663                                 {
1664                                         mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0]));
1665                                 }
1666                         }
1667                         if (0 != group->l2_cache_core_ref_count[1])
1668                         {
1669                                 if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[1])) &&
1670                                                                         (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[1])))
1671                                 {
1672                                         mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[1]));
1673                                 }
1674                         }
1675
1676 #endif
1677                 }
1678                 else
1679                 {
1680                         /* update performance counters for a physical group's pp core */
1681                         mali_pp_update_performance_counters(group->pp_core, group->pp_running_job, group->pp_running_sub_job);
1682
1683 #if defined(CONFIG_MALI400_PROFILING)
1684                         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP|
1685                                                       MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(group->pp_core))|
1686                                                       MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL,
1687                                                       mali_pp_job_get_perf_counter_value0(group->pp_running_job, group->pp_running_sub_job),
1688                                                       mali_pp_job_get_perf_counter_value1(group->pp_running_job, group->pp_running_sub_job),
1689                                                       mali_pp_job_get_perf_counter_src0(group->pp_running_job) | (mali_pp_job_get_perf_counter_src1(group->pp_running_job) << 8),
1690                                                       0, 0);
1691                         if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) &&
1692                                         (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0])))
1693                         {
1694                                 mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0]));
1695                         }
1696 #endif
1697                 }
1698         }
1699 }
1700
1701 static void mali_group_timeout(void *data)
1702 {
1703         struct mali_group *group = (struct mali_group *)data;
1704
1705         group->core_timed_out = MALI_TRUE;
1706
1707         if (NULL != group->gp_core)
1708         {
1709                 MALI_DEBUG_PRINT(2, ("Mali group: TIMEOUT on %s\n", mali_gp_get_hw_core_desc(group->gp_core)));
1710                 _mali_osk_wq_schedule_work(group->bottom_half_work_gp);
1711         }
1712         else
1713         {
1714                 MALI_DEBUG_PRINT(2, ("Mali group: TIMEOUT on %s\n", mali_pp_get_hw_core_desc(group->pp_core)));
1715                 _mali_osk_wq_schedule_work(group->bottom_half_work_pp);
1716         }
1717 }
1718
1719 void mali_group_zap_session(struct mali_group *group, struct mali_session_data *session)
1720 {
1721         MALI_DEBUG_ASSERT_POINTER(group);
1722         MALI_DEBUG_ASSERT_POINTER(session);
1723
1724         /* Early out - safe even if mutex is not held */
1725         if (group->session != session) return;
1726
1727         mali_group_lock(group);
1728
1729         mali_group_remove_session_if_unused(group, session);
1730
1731         if (group->session == session)
1732         {
1733                 /* The Zap also does the stall and disable_stall */
1734                 mali_bool zap_success = mali_mmu_zap_tlb(group->mmu);
1735                 if (MALI_TRUE != zap_success)
1736                 {
1737                         MALI_DEBUG_PRINT(2, ("Mali memory unmap failed. Doing pagefault handling.\n"));
1738                         mali_group_mmu_page_fault(group);
1739                 }
1740         }
1741
1742         mali_group_unlock(group);
1743 }
1744
1745 #if defined(CONFIG_MALI400_PROFILING)
1746 static void mali_group_report_l2_cache_counters_per_core(struct mali_group *group, u32 core_num)
1747 {
1748         u32 source0 = 0;
1749         u32 value0 = 0;
1750         u32 source1 = 0;
1751         u32 value1 = 0;
1752         u32 profiling_channel = 0;
1753
1754         switch(core_num)
1755         {
1756                 case 0: profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE |
1757                                 MALI_PROFILING_EVENT_CHANNEL_GPU |
1758                                 MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L20_COUNTERS;
1759                                 break;
1760                 case 1: profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE |
1761                                 MALI_PROFILING_EVENT_CHANNEL_GPU |
1762                                 MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L21_COUNTERS;
1763                                 break;
1764                 case 2: profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE |
1765                                 MALI_PROFILING_EVENT_CHANNEL_GPU |
1766                                 MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L22_COUNTERS;
1767                                 break;
1768                 default: profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE |
1769                                 MALI_PROFILING_EVENT_CHANNEL_GPU |
1770                                 MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L20_COUNTERS;
1771                                 break;
1772         }
1773
1774         if (0 == core_num)
1775         {
1776                 mali_l2_cache_core_get_counter_values(group->l2_cache_core[0], &source0, &value0, &source1, &value1);
1777         }
1778         if (1 == core_num)
1779         {
1780                 if (1 == mali_l2_cache_get_id(group->l2_cache_core[0]))
1781                 {
1782                         mali_l2_cache_core_get_counter_values(group->l2_cache_core[0], &source0, &value0, &source1, &value1);
1783                 }
1784                 else if (1 == mali_l2_cache_get_id(group->l2_cache_core[1]))
1785                 {
1786                         mali_l2_cache_core_get_counter_values(group->l2_cache_core[1], &source0, &value0, &source1, &value1);
1787                 }
1788         }
1789         if (2 == core_num)
1790         {
1791                 if (2 == mali_l2_cache_get_id(group->l2_cache_core[0]))
1792                 {
1793                         mali_l2_cache_core_get_counter_values(group->l2_cache_core[0], &source0, &value0, &source1, &value1);
1794                 }
1795                 else if (2 == mali_l2_cache_get_id(group->l2_cache_core[1]))
1796                 {
1797                         mali_l2_cache_core_get_counter_values(group->l2_cache_core[1], &source0, &value0, &source1, &value1);
1798                 }
1799         }
1800
1801         _mali_osk_profiling_add_event(profiling_channel, source1 << 8 | source0, value0, value1, 0, 0);
1802 }
1803 #endif /* #if defined(CONFIG_MALI400_PROFILING) */