drivers/gpu/arm/mali400/r4p0_rel0/common/mali_kernel_core.c

   1 /*
   2  * Copyright (C) 2010-2012 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_kernel_common.h"
  12 #include "mali_session.h"
  13 #include "mali_osk.h"
  14 #include "mali_osk_mali.h"
  15 #include "mali_ukk.h"
  16 #include "mali_kernel_core.h"
  17 #include "mali_memory.h"
  18 #include "mali_mem_validation.h"
  19 #include "mali_mmu.h"
  20 #include "mali_mmu_page_directory.h"
  21 #include "mali_dlbu.h"
  22 #include "mali_broadcast.h"
  23 #include "mali_gp.h"
  24 #include "mali_pp.h"
  25 #include "mali_gp_scheduler.h"
  26 #include "mali_pp_scheduler.h"
  27 #include "mali_pp_job.h"
  28 #include "mali_group.h"
  29 #include "mali_pm.h"
  30 #include "mali_pmu.h"
  31 #include "mali_scheduler.h"
  32 #include "mali_kernel_utilization.h"
  33 #include "mali_l2_cache.h"
  34 #include "mali_dma.h"
  35 #include "mali_timeline.h"
  36 #include "mali_soft_job.h"
  37 #include "mali_pm_domain.h"
  38 #if defined(CONFIG_MALI400_PROFILING)
  39 #include "mali_osk_profiling.h"
  40 #endif
  41 #if defined(CONFIG_MALI400_INTERNAL_PROFILING)
  42 #include "mali_profiling_internal.h"
  43 #endif
  44
  45
  46 /* Mali GPU memory. Real values come from module parameter or from device specific data */
  47 unsigned int mali_dedicated_mem_start = 0;
  48 unsigned int mali_dedicated_mem_size = 0;
  49 unsigned int mali_shared_mem_size = 0;
  50
  51 /* Frame buffer memory to be accessible by Mali GPU */
  52 int mali_fb_start = 0;
  53 int mali_fb_size = 0;
  54
  55 /* Mali max job runtime */
  56 extern int mali_max_job_runtime;
  57
  58 /** Start profiling from module load? */
  59 int mali_boot_profiling = 0;
  60
  61 /** Limits for the number of PP cores behind each L2 cache. */
  62 int mali_max_pp_cores_group_1 = 0xFF;
  63 int mali_max_pp_cores_group_2 = 0xFF;
  64
  65 int mali_inited_pp_cores_group_1 = 0;
  66 int mali_inited_pp_cores_group_2 = 0;
  67
  68 static _mali_product_id_t global_product_id = _MALI_PRODUCT_ID_UNKNOWN;
  69 static u32 global_gpu_base_address = 0;
  70 static u32 global_gpu_major_version = 0;
  71 static u32 global_gpu_minor_version = 0;
  72
  73 mali_bool mali_gpu_class_is_mali450 = MALI_FALSE;
  74
  75 static _mali_osk_errcode_t mali_set_global_gpu_base_address(void)
  76 {
  77         global_gpu_base_address = _mali_osk_resource_base_address();
  78         if (0 == global_gpu_base_address) {
  79                 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
  80         }
  81
  82         return _MALI_OSK_ERR_OK;
  83 }
  84
  85 static u32 mali_get_bcast_id(_mali_osk_resource_t *resource_pp)
  86 {
  87         switch (resource_pp->base - global_gpu_base_address) {
  88         case 0x08000:
  89         case 0x20000: /* fall-through for aliased mapping */
  90                 return 0x01;
  91         case 0x0A000:
  92         case 0x22000: /* fall-through for aliased mapping */
  93                 return 0x02;
  94         case 0x0C000:
  95         case 0x24000: /* fall-through for aliased mapping */
  96                 return 0x04;
  97         case 0x0E000:
  98         case 0x26000: /* fall-through for aliased mapping */
  99                 return 0x08;
 100         case 0x28000:
 101                 return 0x10;
 102         case 0x2A000:
 103                 return 0x20;
 104         case 0x2C000:
 105                 return 0x40;
 106         case 0x2E000:
 107                 return 0x80;
 108         default:
 109                 return 0;
 110         }
 111 }
 112
 113 static _mali_osk_errcode_t mali_parse_product_info(void)
 114 {
 115         /*
 116          * Mali-200 has the PP core first, while Mali-300, Mali-400 and Mali-450 have the GP core first.
 117          * Look at the version register for the first PP core in order to determine the GPU HW revision.
 118          */
 119
 120         u32 first_pp_offset;
 121         _mali_osk_resource_t first_pp_resource;
 122
 123         /* Find out where the first PP core is located */
 124         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x8000, NULL)) {
 125                 /* Mali-300/400/450 */
 126                 first_pp_offset = 0x8000;
 127         } else {
 128                 /* Mali-200 */
 129                 first_pp_offset = 0x0000;
 130         }
 131
 132         /* Find the first PP core resource (again) */
 133         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + first_pp_offset, &first_pp_resource)) {
 134                 /* Create a dummy PP object for this core so that we can read the version register */
 135                 struct mali_group *group = mali_group_create(NULL, NULL, NULL);
 136                 if (NULL != group) {
 137                         struct mali_pp_core *pp_core = mali_pp_create(&first_pp_resource, group, MALI_FALSE, mali_get_bcast_id(&first_pp_resource));
 138                         if (NULL != pp_core) {
 139                                 u32 pp_version = mali_pp_core_get_version(pp_core);
 140                                 mali_group_delete(group);
 141
 142                                 global_gpu_major_version = (pp_version >> 8) & 0xFF;
 143                                 global_gpu_minor_version = pp_version & 0xFF;
 144
 145                                 switch (pp_version >> 16) {
 146                                 case MALI200_PP_PRODUCT_ID:
 147                                         global_product_id = _MALI_PRODUCT_ID_MALI200;
 148                                         MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-200 r%up%u\n", global_gpu_major_version, global_gpu_minor_version));
 149                                         MALI_PRINT_ERROR(("Mali-200 is not supported by this driver.\n"));
 150                                         _mali_osk_abort();
 151                                         break;
 152                                 case MALI300_PP_PRODUCT_ID:
 153                                         global_product_id = _MALI_PRODUCT_ID_MALI300;
 154                                         MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-300 r%up%u\n", global_gpu_major_version, global_gpu_minor_version));
 155                                         break;
 156                                 case MALI400_PP_PRODUCT_ID:
 157                                         global_product_id = _MALI_PRODUCT_ID_MALI400;
 158                                         MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-400 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version));
 159                                         break;
 160                                 case MALI450_PP_PRODUCT_ID:
 161                                         global_product_id = _MALI_PRODUCT_ID_MALI450;
 162                                         MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-450 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version));
 163                                         break;
 164                                 default:
 165                                         MALI_DEBUG_PRINT(2, ("Found unknown Mali GPU (r%up%u)\n", global_gpu_major_version, global_gpu_minor_version));
 166                                         return _MALI_OSK_ERR_FAULT;
 167                                 }
 168
 169                                 return _MALI_OSK_ERR_OK;
 170                         } else {
 171                                 MALI_PRINT_ERROR(("Failed to create initial PP object\n"));
 172                         }
 173                 } else {
 174                         MALI_PRINT_ERROR(("Failed to create initial group object\n"));
 175                 }
 176         } else {
 177                 MALI_PRINT_ERROR(("First PP core not specified in config file\n"));
 178         }
 179
 180         return _MALI_OSK_ERR_FAULT;
 181 }
 182
 183
 184 static void mali_resource_count(u32 *pp_count, u32 *l2_count)
 185 {
 186         *pp_count = 0;
 187         *l2_count = 0;
 188
 189         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x08000, NULL)) {
 190                 ++(*pp_count);
 191         }
 192         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x0A000, NULL)) {
 193                 ++(*pp_count);
 194         }
 195         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x0C000, NULL)) {
 196                 ++(*pp_count);
 197         }
 198         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x0E000, NULL)) {
 199                 ++(*pp_count);
 200         }
 201         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x28000, NULL)) {
 202                 ++(*pp_count);
 203         }
 204         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x2A000, NULL)) {
 205                 ++(*pp_count);
 206         }
 207         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x2C000, NULL)) {
 208                 ++(*pp_count);
 209         }
 210         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x2E000, NULL)) {
 211                 ++(*pp_count);
 212         }
 213
 214         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x1000, NULL)) {
 215                 ++(*l2_count);
 216         }
 217         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x10000, NULL)) {
 218                 ++(*l2_count);
 219         }
 220         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x11000, NULL)) {
 221                 ++(*l2_count);
 222         }
 223 }
 224
 225 static void mali_delete_groups(void)
 226 {
 227         struct mali_group *group;
 228
 229         group = mali_group_get_glob_group(0);
 230         while (NULL != group) {
 231                 mali_group_delete(group);
 232                 group = mali_group_get_glob_group(0);
 233         }
 234
 235         MALI_DEBUG_ASSERT(0 == mali_group_get_glob_num_groups());
 236 }
 237
 238 static void mali_delete_l2_cache_cores(void)
 239 {
 240         struct mali_l2_cache_core *l2;
 241
 242         l2 = mali_l2_cache_core_get_glob_l2_core(0);
 243         while (NULL != l2) {
 244                 mali_l2_cache_delete(l2);
 245                 l2 = mali_l2_cache_core_get_glob_l2_core(0);
 246         }
 247
 248         MALI_DEBUG_ASSERT(0 == mali_l2_cache_core_get_glob_num_l2_cores());
 249 }
 250
 251 static struct mali_l2_cache_core *mali_create_l2_cache_core(_mali_osk_resource_t *resource)
 252 {
 253         struct mali_l2_cache_core *l2_cache = NULL;
 254
 255         if (NULL != resource) {
 256
 257                 MALI_DEBUG_PRINT(3, ("Found L2 cache %s\n", resource->description));
 258
 259                 l2_cache = mali_l2_cache_create(resource);
 260                 if (NULL == l2_cache) {
 261                         MALI_PRINT_ERROR(("Failed to create L2 cache object\n"));
 262                         return NULL;
 263                 }
 264         }
 265         MALI_DEBUG_PRINT(3, ("Created L2 cache core object\n"));
 266
 267         return l2_cache;
 268 }
 269
 270 static _mali_osk_errcode_t mali_parse_config_l2_cache(void)
 271 {
 272         struct mali_l2_cache_core *l2_cache = NULL;
 273
 274         if (mali_is_mali400()) {
 275                 _mali_osk_resource_t l2_resource;
 276                 if (_MALI_OSK_ERR_OK != _mali_osk_resource_find(global_gpu_base_address + 0x1000, &l2_resource)) {
 277                         MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache in config file\n"));
 278                         return _MALI_OSK_ERR_FAULT;
 279                 }
 280
 281                 l2_cache = mali_create_l2_cache_core(&l2_resource);
 282                 if (NULL == l2_cache) {
 283                         return _MALI_OSK_ERR_FAULT;
 284                 }
 285                 mali_pm_domain_add_l2(mali_pmu_get_domain_mask(MALI_L20_DOMAIN_INDEX), l2_cache);
 286         } else if (mali_is_mali450()) {
 287                 /*
 288                  * L2 for GP    at 0x10000
 289                  * L2 for PP0-3 at 0x01000
 290                  * L2 for PP4-7 at 0x11000 (optional)
 291                  */
 292
 293                 _mali_osk_resource_t l2_gp_resource;
 294                 _mali_osk_resource_t l2_pp_grp0_resource;
 295                 _mali_osk_resource_t l2_pp_grp1_resource;
 296
 297                 /* Make cluster for GP's L2 */
 298                 if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x10000, &l2_gp_resource)) {
 299                         MALI_DEBUG_PRINT(3, ("Creating Mali-450 L2 cache core for GP\n"));
 300                         l2_cache = mali_create_l2_cache_core(&l2_gp_resource);
 301                         if (NULL == l2_cache) {
 302                                 return _MALI_OSK_ERR_FAULT;
 303                         }
 304                         mali_pm_domain_add_l2(mali_pmu_get_domain_mask(MALI_L20_DOMAIN_INDEX), l2_cache);
 305                 } else {
 306                         MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache for GP in config file\n"));
 307                         return _MALI_OSK_ERR_FAULT;
 308                 }
 309
 310                 /* Find corresponding l2 domain */
 311                 if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x1000, &l2_pp_grp0_resource)) {
 312                         MALI_DEBUG_PRINT(3, ("Creating Mali-450 L2 cache core for PP group 0\n"));
 313                         l2_cache = mali_create_l2_cache_core(&l2_pp_grp0_resource);
 314                         if (NULL == l2_cache) {
 315                                 return _MALI_OSK_ERR_FAULT;
 316                         }
 317                         mali_pm_domain_add_l2(mali_pmu_get_domain_mask(MALI_L21_DOMAIN_INDEX), l2_cache);
 318                 } else {
 319                         MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache for PP group 0 in config file\n"));
 320                         return _MALI_OSK_ERR_FAULT;
 321                 }
 322
 323                 /* Second PP core group is optional, don't fail if we don't find it */
 324                 if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x11000, &l2_pp_grp1_resource)) {
 325                         MALI_DEBUG_PRINT(3, ("Creating Mali-450 L2 cache core for PP group 1\n"));
 326                         l2_cache = mali_create_l2_cache_core(&l2_pp_grp1_resource);
 327                         if (NULL == l2_cache) {
 328                                 return _MALI_OSK_ERR_FAULT;
 329                         }
 330                         mali_pm_domain_add_l2(mali_pmu_get_domain_mask(MALI_L22_DOMAIN_INDEX), l2_cache);
 331                 }
 332         }
 333
 334         return _MALI_OSK_ERR_OK;
 335 }
 336
 337 static struct mali_group *mali_create_group(struct mali_l2_cache_core *cache,
 338         _mali_osk_resource_t *resource_mmu,
 339         _mali_osk_resource_t *resource_gp,
 340         _mali_osk_resource_t *resource_pp)
 341 {
 342         struct mali_mmu_core *mmu;
 343         struct mali_group *group;
 344
 345         MALI_DEBUG_PRINT(3, ("Starting new group for MMU %s\n", resource_mmu->description));
 346
 347         /* Create the group object */
 348         group = mali_group_create(cache, NULL, NULL);
 349         if (NULL == group) {
 350                 MALI_PRINT_ERROR(("Failed to create group object for MMU %s\n", resource_mmu->description));
 351                 return NULL;
 352         }
 353
 354         /* Create the MMU object inside group */
 355         mmu = mali_mmu_create(resource_mmu, group, MALI_FALSE);
 356         if (NULL == mmu) {
 357                 MALI_PRINT_ERROR(("Failed to create MMU object\n"));
 358                 mali_group_delete(group);
 359                 return NULL;
 360         }
 361
 362         if (NULL != resource_gp) {
 363                 /* Create the GP core object inside this group */
 364                 struct mali_gp_core *gp_core = mali_gp_create(resource_gp, group);
 365                 if (NULL == gp_core) {
 366                         /* No need to clean up now, as we will clean up everything linked in from the cluster when we fail this function */
 367                         MALI_PRINT_ERROR(("Failed to create GP object\n"));
 368                         mali_group_delete(group);
 369                         return NULL;
 370                 }
 371         }
 372
 373         if (NULL != resource_pp) {
 374                 struct mali_pp_core *pp_core;
 375
 376                 /* Create the PP core object inside this group */
 377                 pp_core = mali_pp_create(resource_pp, group, MALI_FALSE, mali_get_bcast_id(resource_pp));
 378                 if (NULL == pp_core) {
 379                         /* No need to clean up now, as we will clean up everything linked in from the cluster when we fail this function */
 380                         MALI_PRINT_ERROR(("Failed to create PP object\n"));
 381                         mali_group_delete(group);
 382                         return NULL;
 383                 }
 384         }
 385
 386         /* Reset group */
 387         mali_group_lock(group);
 388         mali_group_reset(group);
 389         mali_group_unlock(group);
 390
 391         return group;
 392 }
 393
 394 static _mali_osk_errcode_t mali_create_virtual_group(_mali_osk_resource_t *resource_mmu_pp_bcast,
 395         _mali_osk_resource_t *resource_pp_bcast,
 396         _mali_osk_resource_t *resource_dlbu,
 397         _mali_osk_resource_t *resource_bcast)
 398 {
 399         struct mali_mmu_core *mmu_pp_bcast_core;
 400         struct mali_pp_core *pp_bcast_core;
 401         struct mali_dlbu_core *dlbu_core;
 402         struct mali_bcast_unit *bcast_core;
 403         struct mali_group *group;
 404
 405         MALI_DEBUG_PRINT(2, ("Starting new virtual group for MMU PP broadcast core %s\n", resource_mmu_pp_bcast->description));
 406
 407         /* Create the DLBU core object */
 408         dlbu_core = mali_dlbu_create(resource_dlbu);
 409         if (NULL == dlbu_core) {
 410                 MALI_PRINT_ERROR(("Failed to create DLBU object \n"));
 411                 return _MALI_OSK_ERR_FAULT;
 412         }
 413
 414         /* Create the Broadcast unit core */
 415         bcast_core = mali_bcast_unit_create(resource_bcast);
 416         if (NULL == bcast_core) {
 417                 MALI_PRINT_ERROR(("Failed to create Broadcast unit object!\n"));
 418                 mali_dlbu_delete(dlbu_core);
 419                 return _MALI_OSK_ERR_FAULT;
 420         }
 421
 422         /* Create the group object */
 423         group = mali_group_create(NULL, dlbu_core, bcast_core);
 424         if (NULL == group) {
 425                 MALI_PRINT_ERROR(("Failed to create group object for MMU PP broadcast core %s\n", resource_mmu_pp_bcast->description));
 426                 mali_bcast_unit_delete(bcast_core);
 427                 mali_dlbu_delete(dlbu_core);
 428                 return _MALI_OSK_ERR_FAULT;
 429         }
 430
 431         /* Create the MMU object inside group */
 432         mmu_pp_bcast_core = mali_mmu_create(resource_mmu_pp_bcast, group, MALI_TRUE);
 433         if (NULL == mmu_pp_bcast_core) {
 434                 MALI_PRINT_ERROR(("Failed to create MMU PP broadcast object\n"));
 435                 mali_group_delete(group);
 436                 return _MALI_OSK_ERR_FAULT;
 437         }
 438
 439         /* Create the PP core object inside this group */
 440         pp_bcast_core = mali_pp_create(resource_pp_bcast, group, MALI_TRUE, 0);
 441         if (NULL == pp_bcast_core) {
 442                 /* No need to clean up now, as we will clean up everything linked in from the cluster when we fail this function */
 443                 MALI_PRINT_ERROR(("Failed to create PP object\n"));
 444                 mali_group_delete(group);
 445                 return _MALI_OSK_ERR_FAULT;
 446         }
 447
 448         return _MALI_OSK_ERR_OK;
 449 }
 450
 451 static _mali_osk_errcode_t mali_parse_config_groups(void)
 452 {
 453         struct mali_group *group;
 454         int cluster_id_gp = 0;
 455         int cluster_id_pp_grp0 = 0;
 456         int cluster_id_pp_grp1 = 0;
 457         int i;
 458
 459         _mali_osk_resource_t resource_gp;
 460         _mali_osk_resource_t resource_gp_mmu;
 461         _mali_osk_resource_t resource_pp[8];
 462         _mali_osk_resource_t resource_pp_mmu[8];
 463         _mali_osk_resource_t resource_pp_mmu_bcast;
 464         _mali_osk_resource_t resource_pp_bcast;
 465         _mali_osk_resource_t resource_dlbu;
 466         _mali_osk_resource_t resource_bcast;
 467         _mali_osk_errcode_t resource_gp_found;
 468         _mali_osk_errcode_t resource_gp_mmu_found;
 469         _mali_osk_errcode_t resource_pp_found[8];
 470         _mali_osk_errcode_t resource_pp_mmu_found[8];
 471         _mali_osk_errcode_t resource_pp_mmu_bcast_found;
 472         _mali_osk_errcode_t resource_pp_bcast_found;
 473         _mali_osk_errcode_t resource_dlbu_found;
 474         _mali_osk_errcode_t resource_bcast_found;
 475
 476         if (!(mali_is_mali400() || mali_is_mali450())) {
 477                 /* No known HW core */
 478                 return _MALI_OSK_ERR_FAULT;
 479         }
 480
 481         if (MALI_MAX_JOB_RUNTIME_DEFAULT == mali_max_job_runtime) {
 482                 /* Group settings are not overridden by module parameters, so use device settings */
 483                 struct _mali_osk_device_data data = { 0, };
 484
 485                 if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) {
 486                         /* Use device specific settings (if defined) */
 487                         if (0 != data.max_job_runtime) {
 488                                 mali_max_job_runtime = data.max_job_runtime;
 489                         }
 490                 }
 491         }
 492
 493         if (mali_is_mali450()) {
 494                 /* Mali-450 have separate L2s for GP, and PP core group(s) */
 495                 cluster_id_pp_grp0 = 1;
 496                 cluster_id_pp_grp1 = 2;
 497         }
 498
 499         resource_gp_found = _mali_osk_resource_find(global_gpu_base_address + 0x00000, &resource_gp);
 500         resource_gp_mmu_found = _mali_osk_resource_find(global_gpu_base_address + 0x03000, &resource_gp_mmu);
 501         resource_pp_found[0] = _mali_osk_resource_find(global_gpu_base_address + 0x08000, &(resource_pp[0]));
 502         resource_pp_found[1] = _mali_osk_resource_find(global_gpu_base_address + 0x0A000, &(resource_pp[1]));
 503         resource_pp_found[2] = _mali_osk_resource_find(global_gpu_base_address + 0x0C000, &(resource_pp[2]));
 504         resource_pp_found[3] = _mali_osk_resource_find(global_gpu_base_address + 0x0E000, &(resource_pp[3]));
 505         resource_pp_found[4] = _mali_osk_resource_find(global_gpu_base_address + 0x28000, &(resource_pp[4]));
 506         resource_pp_found[5] = _mali_osk_resource_find(global_gpu_base_address + 0x2A000, &(resource_pp[5]));
 507         resource_pp_found[6] = _mali_osk_resource_find(global_gpu_base_address + 0x2C000, &(resource_pp[6]));
 508         resource_pp_found[7] = _mali_osk_resource_find(global_gpu_base_address + 0x2E000, &(resource_pp[7]));
 509         resource_pp_mmu_found[0] = _mali_osk_resource_find(global_gpu_base_address + 0x04000, &(resource_pp_mmu[0]));
 510         resource_pp_mmu_found[1] = _mali_osk_resource_find(global_gpu_base_address + 0x05000, &(resource_pp_mmu[1]));
 511         resource_pp_mmu_found[2] = _mali_osk_resource_find(global_gpu_base_address + 0x06000, &(resource_pp_mmu[2]));
 512         resource_pp_mmu_found[3] = _mali_osk_resource_find(global_gpu_base_address + 0x07000, &(resource_pp_mmu[3]));
 513         resource_pp_mmu_found[4] = _mali_osk_resource_find(global_gpu_base_address + 0x1C000, &(resource_pp_mmu[4]));
 514         resource_pp_mmu_found[5] = _mali_osk_resource_find(global_gpu_base_address + 0x1D000, &(resource_pp_mmu[5]));
 515         resource_pp_mmu_found[6] = _mali_osk_resource_find(global_gpu_base_address + 0x1E000, &(resource_pp_mmu[6]));
 516         resource_pp_mmu_found[7] = _mali_osk_resource_find(global_gpu_base_address + 0x1F000, &(resource_pp_mmu[7]));
 517
 518
 519         if (mali_is_mali450()) {
 520                 resource_bcast_found = _mali_osk_resource_find(global_gpu_base_address + 0x13000, &resource_bcast);
 521                 resource_dlbu_found = _mali_osk_resource_find(global_gpu_base_address + 0x14000, &resource_dlbu);
 522                 resource_pp_mmu_bcast_found = _mali_osk_resource_find(global_gpu_base_address + 0x15000, &resource_pp_mmu_bcast);
 523                 resource_pp_bcast_found = _mali_osk_resource_find(global_gpu_base_address + 0x16000, &resource_pp_bcast);
 524
 525                 if (_MALI_OSK_ERR_OK != resource_bcast_found ||
 526                     _MALI_OSK_ERR_OK != resource_dlbu_found ||
 527                     _MALI_OSK_ERR_OK != resource_pp_mmu_bcast_found ||
 528                     _MALI_OSK_ERR_OK != resource_pp_bcast_found) {
 529                         /* Missing mandatory core(s) for Mali-450 */
 530                         MALI_DEBUG_PRINT(2, ("Missing mandatory resources, Mali-450 needs DLBU, Broadcast unit, virtual PP core and virtual MMU\n"));
 531                         return _MALI_OSK_ERR_FAULT;
 532                 }
 533         }
 534
 535         if (_MALI_OSK_ERR_OK != resource_gp_found ||
 536             _MALI_OSK_ERR_OK != resource_gp_mmu_found ||
 537             _MALI_OSK_ERR_OK != resource_pp_found[0] ||
 538             _MALI_OSK_ERR_OK != resource_pp_mmu_found[0]) {
 539                 /* Missing mandatory core(s) */
 540                 MALI_DEBUG_PRINT(2, ("Missing mandatory resource, need at least one GP and one PP, both with a separate MMU\n"));
 541                 return _MALI_OSK_ERR_FAULT;
 542         }
 543
 544         MALI_DEBUG_ASSERT(1 <= mali_l2_cache_core_get_glob_num_l2_cores());
 545         group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_gp), &resource_gp_mmu, &resource_gp, NULL);
 546         if (NULL == group) {
 547                 return _MALI_OSK_ERR_FAULT;
 548         }
 549
 550         /* Add GP in group, for PMU ref count */
 551         mali_pm_domain_add_group(mali_pmu_get_domain_mask(MALI_GP_DOMAIN_INDEX), group);
 552
 553         /* Create group for first (and mandatory) PP core */
 554         MALI_DEBUG_ASSERT(mali_l2_cache_core_get_glob_num_l2_cores() >= (cluster_id_pp_grp0 + 1)); /* >= 1 on Mali-300 and Mali-400, >= 2 on Mali-450 */
 555         group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_pp_grp0), &resource_pp_mmu[0], NULL, &resource_pp[0]);
 556         if (NULL == group) {
 557                 return _MALI_OSK_ERR_FAULT;
 558         }
 559
 560         /* Find corresponding pp domain */
 561         mali_pm_domain_add_group(mali_pmu_get_domain_mask(MALI_PP0_DOMAIN_INDEX), group);
 562
 563         mali_inited_pp_cores_group_1++;
 564
 565         /* Create groups for rest of the cores in the first PP core group */
 566         for (i = 1; i < 4; i++) { /* First half of the PP cores belong to first core group */
 567                 if (mali_inited_pp_cores_group_1 < mali_max_pp_cores_group_1) {
 568                         if (_MALI_OSK_ERR_OK == resource_pp_found[i] && _MALI_OSK_ERR_OK == resource_pp_mmu_found[i]) {
 569                                 group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_pp_grp0), &resource_pp_mmu[i], NULL, &resource_pp[i]);
 570                                 if (NULL == group) {
 571                                         return _MALI_OSK_ERR_FAULT;
 572                                 }
 573
 574                                 mali_pm_domain_add_group(mali_pmu_get_domain_mask(i + MALI_PP0_DOMAIN_INDEX), group);
 575
 576                                 mali_inited_pp_cores_group_1++;
 577                         }
 578                 }
 579         }
 580
 581         /* Create groups for cores in the second PP core group */
 582         for (i = 4; i < 8; i++) { /* Second half of the PP cores belong to second core group */
 583                 if (mali_inited_pp_cores_group_2 < mali_max_pp_cores_group_2) {
 584                         if (_MALI_OSK_ERR_OK == resource_pp_found[i] && _MALI_OSK_ERR_OK == resource_pp_mmu_found[i]) {
 585                                 MALI_DEBUG_ASSERT(mali_l2_cache_core_get_glob_num_l2_cores() >= 2); /* Only Mali-450 have a second core group */
 586                                 group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_pp_grp1), &resource_pp_mmu[i], NULL, &resource_pp[i]);
 587                                 if (NULL == group) {
 588                                         return _MALI_OSK_ERR_FAULT;
 589                                 }
 590                                 mali_pm_domain_add_group(mali_pmu_get_domain_mask(i + MALI_PP0_DOMAIN_INDEX), group);
 591                                 mali_inited_pp_cores_group_2++;
 592                         }
 593                 }
 594         }
 595
 596         if(mali_is_mali450()) {
 597                 _mali_osk_errcode_t err = mali_create_virtual_group(&resource_pp_mmu_bcast, &resource_pp_bcast, &resource_dlbu, &resource_bcast);
 598                 if (_MALI_OSK_ERR_OK != err) {
 599                         return err;
 600                 }
 601         }
 602
 603         mali_max_pp_cores_group_1 = mali_inited_pp_cores_group_1;
 604         mali_max_pp_cores_group_2 = mali_inited_pp_cores_group_2;
 605         MALI_DEBUG_PRINT(2, ("%d+%d PP cores initialized\n", mali_inited_pp_cores_group_1, mali_inited_pp_cores_group_2));
 606
 607         return _MALI_OSK_ERR_OK;
 608 }
 609
 610 static _mali_osk_errcode_t mali_check_shared_interrupts(void)
 611 {
 612 #if !defined(CONFIG_MALI_SHARED_INTERRUPTS)
 613         if (MALI_TRUE == _mali_osk_shared_interrupts()) {
 614                 MALI_PRINT_ERROR(("Shared interrupts detected, but driver support is not enabled\n"));
 615                 return _MALI_OSK_ERR_FAULT;
 616         }
 617 #endif /* !defined(CONFIG_MALI_SHARED_INTERRUPTS) */
 618
 619         /* It is OK to compile support for shared interrupts even if Mali is not using it. */
 620         return _MALI_OSK_ERR_OK;
 621 }
 622
 623 static _mali_osk_errcode_t mali_create_pm_domains(void)
 624 {
 625         int i;
 626
 627         for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) {
 628                 if (0x0 == mali_pmu_get_domain_mask(i)) continue;
 629
 630                 if (NULL == mali_pm_domain_create(mali_pmu_get_domain_mask(i))) {
 631                         return _MALI_OSK_ERR_NOMEM;
 632                 }
 633         }
 634
 635         return _MALI_OSK_ERR_OK;
 636 }
 637
 638 static void mali_use_default_pm_domain_config(void)
 639 {
 640         u32 pp_count_gr1 = 0;
 641         u32 pp_count_gr2 = 0;
 642         u32 l2_count = 0;
 643
 644         MALI_DEBUG_ASSERT(0 != global_gpu_base_address);
 645
 646         /* GP core */
 647         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x00000, NULL)) {
 648                 mali_pmu_set_domain_mask(MALI_GP_DOMAIN_INDEX, 0x01);
 649         }
 650
 651         /* PP0 - PP3 core */
 652         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x08000, NULL)) {
 653                 ++pp_count_gr1;
 654
 655                 if (mali_is_mali400()) {
 656                         mali_pmu_set_domain_mask(MALI_PP0_DOMAIN_INDEX, 0x01<<2);
 657                 } else if (mali_is_mali450()) {
 658                         mali_pmu_set_domain_mask(MALI_PP0_DOMAIN_INDEX, 0x01<<1);
 659                 }
 660         }
 661
 662         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x0A000, NULL)) {
 663                 ++pp_count_gr1;
 664
 665                 if (mali_is_mali400()) {
 666                         mali_pmu_set_domain_mask(MALI_PP1_DOMAIN_INDEX, 0x01<<3);
 667                 } else if (mali_is_mali450()) {
 668                         mali_pmu_set_domain_mask(MALI_PP1_DOMAIN_INDEX, 0x01<<2);
 669                 }
 670         }
 671
 672         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x0C000, NULL)) {
 673                 ++pp_count_gr1;
 674
 675                 if (mali_is_mali400()) {
 676                         mali_pmu_set_domain_mask(MALI_PP2_DOMAIN_INDEX, 0x01<<4);
 677                 } else if (mali_is_mali450()) {
 678                         mali_pmu_set_domain_mask(MALI_PP2_DOMAIN_INDEX, 0x01<<2);
 679                 }
 680         }
 681
 682         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x0E000, NULL)) {
 683                 ++pp_count_gr1;
 684
 685                 if (mali_is_mali400()) {
 686                         mali_pmu_set_domain_mask(MALI_PP3_DOMAIN_INDEX, 0x01<<5);
 687                 } else if (mali_is_mali450()) {
 688                         mali_pmu_set_domain_mask(MALI_PP3_DOMAIN_INDEX, 0x01<<2);
 689                 }
 690         }
 691
 692         /* PP4 - PP7 */
 693         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x28000, NULL)) {
 694                 ++pp_count_gr2;
 695
 696                 mali_pmu_set_domain_mask(MALI_PP4_DOMAIN_INDEX, 0x01<<3);
 697         }
 698
 699         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x2A000, NULL)) {
 700                 ++pp_count_gr2;
 701
 702                 mali_pmu_set_domain_mask(MALI_PP5_DOMAIN_INDEX, 0x01<<3);
 703         }
 704
 705         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x2C000, NULL)) {
 706                 ++pp_count_gr2;
 707
 708                 mali_pmu_set_domain_mask(MALI_PP6_DOMAIN_INDEX, 0x01<<3);
 709         }
 710
 711         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x2E000, NULL)) {
 712                 ++pp_count_gr2;
 713
 714                 mali_pmu_set_domain_mask(MALI_PP7_DOMAIN_INDEX, 0x01<<3);
 715         }
 716
 717         /* L2gp/L2PP0/L2PP4 */
 718         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x10000, NULL)) {
 719                 ++l2_count;
 720
 721                 if (mali_is_mali400()) {
 722                         mali_pmu_set_domain_mask(MALI_L20_DOMAIN_INDEX, 0x01<<1);
 723                 } else if (mali_is_mali450()) {
 724                         mali_pmu_set_domain_mask(MALI_L20_DOMAIN_INDEX, 0x01<<0);
 725                 }
 726         }
 727
 728         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x1000, NULL)) {
 729                 ++l2_count;
 730
 731                 mali_pmu_set_domain_mask(MALI_L21_DOMAIN_INDEX, 0x01<<1);
 732         }
 733
 734         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x11000, NULL)) {
 735                 ++l2_count;
 736
 737                 mali_pmu_set_domain_mask(MALI_L22_DOMAIN_INDEX, 0x01<<3);
 738         }
 739
 740         MALI_DEBUG_PRINT(2, ("Using default PMU domain config: (%d) gr1_pp_cores, (%d) gr2_pp_cores, (%d) l2_count. \n", pp_count_gr1, pp_count_gr2, l2_count));
 741 }
 742
 743 static void mali_set_pmu_global_domain_config(void)
 744 {
 745         struct _mali_osk_device_data data = { 0, };
 746         int i = 0;
 747
 748         if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) {
 749                 /* Check whether has customized pmu domain configure */
 750                 for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) {
 751                         if (0 != data.pmu_domain_config[i]) break;
 752                 }
 753
 754                 if (MALI_MAX_NUMBER_OF_DOMAINS == i) {
 755                         mali_use_default_pm_domain_config();
 756                 } else {
 757                         /* Copy the customer config to global config */
 758                         mali_pmu_copy_domain_mask(data.pmu_domain_config, sizeof(data.pmu_domain_config));
 759                 }
 760         }
 761 }
 762
 763 static _mali_osk_errcode_t mali_parse_config_pmu(void)
 764 {
 765         _mali_osk_resource_t resource_pmu;
 766
 767         MALI_DEBUG_ASSERT(0 != global_gpu_base_address);
 768
 769         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x02000, &resource_pmu)) {
 770                 struct mali_pmu_core *pmu;
 771
 772                 mali_set_pmu_global_domain_config();
 773
 774                 pmu = mali_pmu_create(&resource_pmu);
 775                 if (NULL == pmu) {
 776                         MALI_PRINT_ERROR(("Failed to create PMU\n"));
 777                         return _MALI_OSK_ERR_FAULT;
 778                 }
 779         }
 780
 781         /* It's ok if the PMU doesn't exist */
 782         return _MALI_OSK_ERR_OK;
 783 }
 784
 785 static _mali_osk_errcode_t mali_parse_config_dma(void)
 786 {
 787         _mali_osk_resource_t resource_dma;
 788
 789         if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(global_gpu_base_address + 0x12000, &resource_dma)) {
 790                 if (NULL == mali_dma_create(&resource_dma)) {
 791                         return _MALI_OSK_ERR_FAULT;
 792                 }
 793                 return _MALI_OSK_ERR_OK;
 794         } else {
 795                 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
 796         }
 797 }
 798
 799 static _mali_osk_errcode_t mali_parse_config_memory(void)
 800 {
 801         _mali_osk_errcode_t ret;
 802
 803         if (0 == mali_dedicated_mem_start && 0 == mali_dedicated_mem_size && 0 == mali_shared_mem_size) {
 804                 /* Memory settings are not overridden by module parameters, so use device settings */
 805                 struct _mali_osk_device_data data = { 0, };
 806
 807                 if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) {
 808                         /* Use device specific settings (if defined) */
 809                         mali_dedicated_mem_start = data.dedicated_mem_start;
 810                         mali_dedicated_mem_size = data.dedicated_mem_size;
 811                         mali_shared_mem_size = data.shared_mem_size;
 812                 }
 813
 814                 if (0 == mali_dedicated_mem_start && 0 == mali_dedicated_mem_size && 0 == mali_shared_mem_size) {
 815                         /* No GPU memory specified */
 816                         return _MALI_OSK_ERR_INVALID_ARGS;
 817                 }
 818
 819                 MALI_DEBUG_PRINT(2, ("Using device defined memory settings (dedicated: 0x%08X@0x%08X, shared: 0x%08X)\n",
 820                                      mali_dedicated_mem_size, mali_dedicated_mem_start, mali_shared_mem_size));
 821         } else {
 822                 MALI_DEBUG_PRINT(2, ("Using module defined memory settings (dedicated: 0x%08X@0x%08X, shared: 0x%08X)\n",
 823                                      mali_dedicated_mem_size, mali_dedicated_mem_start, mali_shared_mem_size));
 824         }
 825
 826         if (0 < mali_dedicated_mem_size && 0 != mali_dedicated_mem_start) {
 827                 /* Dedicated memory */
 828                 ret = mali_memory_core_resource_dedicated_memory(mali_dedicated_mem_start, mali_dedicated_mem_size);
 829                 if (_MALI_OSK_ERR_OK != ret) {
 830                         MALI_PRINT_ERROR(("Failed to register dedicated memory\n"));
 831                         mali_memory_terminate();
 832                         return ret;
 833                 }
 834         }
 835
 836         if (0 < mali_shared_mem_size) {
 837                 /* Shared OS memory */
 838                 ret = mali_memory_core_resource_os_memory(mali_shared_mem_size);
 839                 if (_MALI_OSK_ERR_OK != ret) {
 840                         MALI_PRINT_ERROR(("Failed to register shared OS memory\n"));
 841                         mali_memory_terminate();
 842                         return ret;
 843                 }
 844         }
 845
 846         if (0 == mali_fb_start && 0 == mali_fb_size) {
 847                 /* Frame buffer settings are not overridden by module parameters, so use device settings */
 848                 struct _mali_osk_device_data data = { 0, };
 849
 850                 if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) {
 851                         /* Use device specific settings (if defined) */
 852                         mali_fb_start = data.fb_start;
 853                         mali_fb_size = data.fb_size;
 854                 }
 855
 856                 MALI_DEBUG_PRINT(2, ("Using device defined frame buffer settings (0x%08X@0x%08X)\n",
 857                                      mali_fb_size, mali_fb_start));
 858         } else {
 859                 MALI_DEBUG_PRINT(2, ("Using module defined frame buffer settings (0x%08X@0x%08X)\n",
 860                                      mali_fb_size, mali_fb_start));
 861         }
 862
 863         if (0 != mali_fb_size) {
 864                 /* Register frame buffer */
 865                 ret = mali_mem_validation_add_range(mali_fb_start, mali_fb_size);
 866                 if (_MALI_OSK_ERR_OK != ret) {
 867                         MALI_PRINT_ERROR(("Failed to register frame buffer memory region\n"));
 868                         mali_memory_terminate();
 869                         return ret;
 870                 }
 871         }
 872
 873         return _MALI_OSK_ERR_OK;
 874 }
 875
 876 static void mali_detect_gpu_class(void)
 877 {
 878         u32 number_of_pp_cores = 0;
 879         u32 number_of_l2_caches = 0;
 880
 881         mali_resource_count(&number_of_pp_cores, &number_of_l2_caches);
 882         if (number_of_l2_caches > 1) {
 883                 mali_gpu_class_is_mali450 = MALI_TRUE;
 884         }
 885 }
 886
 887 _mali_osk_errcode_t mali_initialize_subsystems(void)
 888 {
 889         _mali_osk_errcode_t err;
 890         struct mali_pmu_core *pmu;
 891
 892         mali_pp_job_initialize();
 893
 894         err = mali_session_initialize();
 895         if (_MALI_OSK_ERR_OK != err) goto session_init_failed;
 896
 897 #if defined(CONFIG_MALI400_PROFILING)
 898         err = _mali_osk_profiling_init(mali_boot_profiling ? MALI_TRUE : MALI_FALSE);
 899         if (_MALI_OSK_ERR_OK != err) {
 900                 /* No biggie if we weren't able to initialize the profiling */
 901                 MALI_PRINT_ERROR(("Failed to initialize profiling, feature will be unavailable\n"));
 902         }
 903 #endif
 904
 905         err = mali_memory_initialize();
 906         if (_MALI_OSK_ERR_OK != err) goto memory_init_failed;
 907
 908         /* Configure memory early. Memory allocation needed for mali_mmu_initialize. */
 909         err = mali_parse_config_memory();
 910         if (_MALI_OSK_ERR_OK != err) goto parse_memory_config_failed;
 911
 912         err = mali_set_global_gpu_base_address();
 913         if (_MALI_OSK_ERR_OK != err) goto set_global_gpu_base_address_failed;
 914
 915         /* Detect gpu class according to l2 cache number */
 916         mali_detect_gpu_class();
 917
 918         err = mali_check_shared_interrupts();
 919         if (_MALI_OSK_ERR_OK != err) goto check_shared_interrupts_failed;
 920
 921         err = mali_pp_scheduler_initialize();
 922         if (_MALI_OSK_ERR_OK != err) goto pp_scheduler_init_failed;
 923
 924         /* Initialize the power management module */
 925         err = mali_pm_initialize();
 926         if (_MALI_OSK_ERR_OK != err) goto pm_init_failed;
 927
 928         /* Initialize the MALI PMU */
 929         err = mali_parse_config_pmu();
 930         if (_MALI_OSK_ERR_OK != err) goto parse_pmu_config_failed;
 931
 932         /* Make sure the power stays on for the rest of this function */
 933         err = _mali_osk_pm_dev_ref_add();
 934         if (_MALI_OSK_ERR_OK != err) goto pm_always_on_failed;
 935
 936         /*
 937          * If run-time PM is used, then the mali_pm module has now already been
 938          * notified that the power now is on (through the resume callback functions).
 939          * However, if run-time PM is not used, then there will probably not be any
 940          * calls to the resume callback functions, so we need to explicitly tell it
 941          * that the power is on.
 942          */
 943         mali_pm_set_power_is_on();
 944
 945         /* Reset PMU HW and ensure all Mali power domains are on */
 946         pmu = mali_pmu_get_global_pmu_core();
 947         if (NULL != pmu) {
 948                 err = mali_pmu_reset(pmu);
 949                 if (_MALI_OSK_ERR_OK != err) goto pmu_reset_failed;
 950         }
 951
 952         /* Detect which Mali GPU we are dealing with */
 953         err = mali_parse_product_info();
 954         if (_MALI_OSK_ERR_OK != err) goto product_info_parsing_failed;
 955
 956         /* The global_product_id is now populated with the correct Mali GPU */
 957
 958         /* Create PM domains only if PMU exists */
 959         if (NULL != pmu) {
 960                 err = mali_create_pm_domains();
 961                 if (_MALI_OSK_ERR_OK != err) goto pm_domain_failed;
 962         }
 963
 964         /* Initialize MMU module */
 965         err = mali_mmu_initialize();
 966         if (_MALI_OSK_ERR_OK != err) goto mmu_init_failed;
 967
 968         if (mali_is_mali450()) {
 969                 err = mali_dlbu_initialize();
 970                 if (_MALI_OSK_ERR_OK != err) goto dlbu_init_failed;
 971
 972                 err = mali_parse_config_dma();
 973                 if (_MALI_OSK_ERR_OK != err) goto dma_parsing_failed;
 974         }
 975
 976         /* Start configuring the actual Mali hardware. */
 977         err = mali_parse_config_l2_cache();
 978         if (_MALI_OSK_ERR_OK != err) goto config_parsing_failed;
 979         err = mali_parse_config_groups();
 980         if (_MALI_OSK_ERR_OK != err) goto config_parsing_failed;
 981
 982         /* Initialize the schedulers */
 983         err = mali_scheduler_initialize();
 984         if (_MALI_OSK_ERR_OK != err) goto scheduler_init_failed;
 985         err = mali_gp_scheduler_initialize();
 986         if (_MALI_OSK_ERR_OK != err) goto gp_scheduler_init_failed;
 987
 988         /* PP scheduler population can't fail */
 989         mali_pp_scheduler_populate();
 990
 991         /* Initialize the GPU utilization tracking */
 992         err = mali_utilization_init();
 993         if (_MALI_OSK_ERR_OK != err) goto utilization_init_failed;
 994
 995         /* Allowing the system to be turned off */
 996         _mali_osk_pm_dev_ref_dec();
 997
 998         MALI_SUCCESS; /* all ok */
 999
1000         /* Error handling */
1001
1002 utilization_init_failed:
1003         mali_pp_scheduler_depopulate();
1004         mali_gp_scheduler_terminate();
1005 gp_scheduler_init_failed:
1006         mali_scheduler_terminate();
1007 scheduler_init_failed:
1008 config_parsing_failed:
1009         mali_delete_groups(); /* Delete any groups not (yet) owned by a scheduler */
1010         mali_delete_l2_cache_cores(); /* Delete L2 cache cores even if config parsing failed. */
1011         {
1012                 struct mali_dma_core *dma = mali_dma_get_global_dma_core();
1013                 if (NULL != dma) mali_dma_delete(dma);
1014         }
1015 dma_parsing_failed:
1016         mali_dlbu_terminate();
1017 dlbu_init_failed:
1018         mali_mmu_terminate();
1019 mmu_init_failed:
1020         mali_pm_domain_terminate();
1021 pm_domain_failed:
1022         /* Nothing to roll back */
1023 product_info_parsing_failed:
1024         /* Nothing to roll back */
1025 pmu_reset_failed:
1026         /* Allowing the system to be turned off */
1027         _mali_osk_pm_dev_ref_dec();
1028 pm_always_on_failed:
1029         pmu = mali_pmu_get_global_pmu_core();
1030         if (NULL != pmu) {
1031                 mali_pmu_delete(pmu);
1032         }
1033 parse_pmu_config_failed:
1034         mali_pm_terminate();
1035 pm_init_failed:
1036         mali_pp_scheduler_terminate();
1037 pp_scheduler_init_failed:
1038 check_shared_interrupts_failed:
1039         global_gpu_base_address = 0;
1040 set_global_gpu_base_address_failed:
1041         /* undoing mali_parse_config_memory() is done by mali_memory_terminate() */
1042 parse_memory_config_failed:
1043         mali_memory_terminate();
1044 memory_init_failed:
1045 #if defined(CONFIG_MALI400_PROFILING)
1046         _mali_osk_profiling_term();
1047 #endif
1048         mali_session_terminate();
1049 session_init_failed:
1050         mali_pp_job_terminate();
1051         return err;
1052 }
1053
1054 void mali_terminate_subsystems(void)
1055 {
1056         struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core();
1057         struct mali_dma_core *dma = mali_dma_get_global_dma_core();
1058
1059         MALI_DEBUG_PRINT(2, ("terminate_subsystems() called\n"));
1060
1061         /* shut down subsystems in reverse order from startup */
1062
1063         /* We need the GPU to be powered up for the terminate sequence */
1064         _mali_osk_pm_dev_ref_add();
1065
1066         mali_utilization_term();
1067         mali_pp_scheduler_depopulate();
1068         mali_gp_scheduler_terminate();
1069         mali_scheduler_terminate();
1070         mali_delete_l2_cache_cores();
1071         if (mali_is_mali450()) {
1072                 mali_dlbu_terminate();
1073         }
1074         mali_mmu_terminate();
1075         if (NULL != pmu) {
1076                 mali_pmu_delete(pmu);
1077         }
1078         if (NULL != dma) {
1079                 mali_dma_delete(dma);
1080         }
1081         mali_pm_terminate();
1082         mali_memory_terminate();
1083 #if defined(CONFIG_MALI400_PROFILING)
1084         _mali_osk_profiling_term();
1085 #endif
1086
1087         /* Allowing the system to be turned off */
1088         _mali_osk_pm_dev_ref_dec();
1089
1090         mali_pp_scheduler_terminate();
1091         mali_session_terminate();
1092
1093         mali_pp_job_terminate();
1094 }
1095
1096 _mali_product_id_t mali_kernel_core_get_product_id(void)
1097 {
1098         return global_product_id;
1099 }
1100
1101 u32 mali_kernel_core_get_gpu_major_version(void)
1102 {
1103         return global_gpu_major_version;
1104 }
1105
1106 u32 mali_kernel_core_get_gpu_minor_version(void)
1107 {
1108         return global_gpu_minor_version;
1109 }
1110
1111 _mali_osk_errcode_t _mali_ukk_get_api_version( _mali_uk_get_api_version_s *args )
1112 {
1113         MALI_DEBUG_ASSERT_POINTER(args);
1114         MALI_CHECK_NON_NULL(args->ctx, _MALI_OSK_ERR_INVALID_ARGS);
1115
1116         /* check compatability */
1117         if ( args->version == _MALI_UK_API_VERSION ) {
1118                 args->compatible = 1;
1119         } else {
1120                 args->compatible = 0;
1121         }
1122
1123         args->version = _MALI_UK_API_VERSION; /* report our version */
1124
1125         /* success regardless of being compatible or not */
1126         MALI_SUCCESS;
1127 }
1128
1129 _mali_osk_errcode_t _mali_ukk_wait_for_notification( _mali_uk_wait_for_notification_s *args )
1130 {
1131         _mali_osk_errcode_t err;
1132         _mali_osk_notification_t * notification;
1133         _mali_osk_notification_queue_t *queue;
1134
1135         /* check input */
1136         MALI_DEBUG_ASSERT_POINTER(args);
1137         MALI_CHECK_NON_NULL(args->ctx, _MALI_OSK_ERR_INVALID_ARGS);
1138
1139         queue = ((struct mali_session_data *)args->ctx)->ioctl_queue;
1140
1141         /* if the queue does not exist we're currently shutting down */
1142         if (NULL == queue) {
1143                 MALI_DEBUG_PRINT(1, ("No notification queue registered with the session. Asking userspace to stop querying\n"));
1144                 args->type = _MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS;
1145                 MALI_SUCCESS;
1146         }
1147
1148         /* receive a notification, might sleep */
1149         err = _mali_osk_notification_queue_receive(queue, &notification);
1150         if (_MALI_OSK_ERR_OK != err) {
1151                 MALI_ERROR(err); /* errcode returned, pass on to caller */
1152         }
1153
1154         /* copy the buffer to the user */
1155         args->type = (_mali_uk_notification_type)notification->notification_type;
1156         _mali_osk_memcpy(&args->data, notification->result_buffer, notification->result_buffer_size);
1157
1158         /* finished with the notification */
1159         _mali_osk_notification_delete( notification );
1160
1161         MALI_SUCCESS; /* all ok */
1162 }
1163
1164 _mali_osk_errcode_t _mali_ukk_post_notification( _mali_uk_post_notification_s *args )
1165 {
1166         _mali_osk_notification_t * notification;
1167         _mali_osk_notification_queue_t *queue;
1168
1169         /* check input */
1170         MALI_DEBUG_ASSERT_POINTER(args);
1171         MALI_CHECK_NON_NULL(args->ctx, _MALI_OSK_ERR_INVALID_ARGS);
1172
1173         queue = ((struct mali_session_data *)args->ctx)->ioctl_queue;
1174
1175         /* if the queue does not exist we're currently shutting down */
1176         if (NULL == queue) {
1177                 MALI_DEBUG_PRINT(1, ("No notification queue registered with the session. Asking userspace to stop querying\n"));
1178                 MALI_SUCCESS;
1179         }
1180
1181         notification = _mali_osk_notification_create(args->type, 0);
1182         if (NULL == notification) {
1183                 MALI_PRINT_ERROR( ("Failed to create notification object\n"));
1184                 return _MALI_OSK_ERR_NOMEM;
1185         }
1186
1187         _mali_osk_notification_queue_send(queue, notification);
1188
1189         MALI_SUCCESS; /* all ok */
1190 }
1191
1192 _mali_osk_errcode_t _mali_ukk_request_high_priority( _mali_uk_request_high_priority_s *args )
1193 {
1194         struct mali_session_data *session;
1195
1196         MALI_DEBUG_ASSERT_POINTER(args);
1197         MALI_CHECK_NON_NULL(args->ctx, _MALI_OSK_ERR_INVALID_ARGS);
1198
1199         session = (struct mali_session_data *) args->ctx;
1200
1201         if (!session->use_high_priority_job_queue) {
1202                 session->use_high_priority_job_queue = MALI_TRUE;
1203                 MALI_DEBUG_PRINT(2, ("Session 0x%08X with pid %d was granted higher priority.\n", session, _mali_osk_get_pid()));
1204         }
1205
1206         MALI_SUCCESS;
1207 }
1208
1209 _mali_osk_errcode_t _mali_ukk_open(void **context)
1210 {
1211         u32 i;
1212         struct mali_session_data *session;
1213
1214         /* allocated struct to track this session */
1215         session = (struct mali_session_data *)_mali_osk_calloc(1, sizeof(struct mali_session_data));
1216         MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_NOMEM);
1217
1218         MALI_DEBUG_PRINT(3, ("Session starting\n"));
1219
1220         /* create a response queue for this session */
1221         session->ioctl_queue = _mali_osk_notification_queue_init();
1222         if (NULL == session->ioctl_queue) {
1223                 _mali_osk_free(session);
1224                 MALI_ERROR(_MALI_OSK_ERR_NOMEM);
1225         }
1226
1227         session->page_directory = mali_mmu_pagedir_alloc();
1228         if (NULL == session->page_directory) {
1229                 _mali_osk_notification_queue_term(session->ioctl_queue);
1230                 _mali_osk_free(session);
1231                 MALI_ERROR(_MALI_OSK_ERR_NOMEM);
1232         }
1233
1234         if (_MALI_OSK_ERR_OK != mali_mmu_pagedir_map(session->page_directory, MALI_DLBU_VIRT_ADDR, _MALI_OSK_MALI_PAGE_SIZE)) {
1235                 MALI_PRINT_ERROR(("Failed to map DLBU page into session\n"));
1236                 _mali_osk_notification_queue_term(session->ioctl_queue);
1237                 _mali_osk_free(session);
1238                 MALI_ERROR(_MALI_OSK_ERR_NOMEM);
1239         }
1240
1241         if (0 != mali_dlbu_phys_addr) {
1242                 mali_mmu_pagedir_update(session->page_directory, MALI_DLBU_VIRT_ADDR, mali_dlbu_phys_addr,
1243                                         _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT);
1244         }
1245
1246         if (_MALI_OSK_ERR_OK != mali_memory_session_begin(session)) {
1247                 mali_mmu_pagedir_free(session->page_directory);
1248                 _mali_osk_notification_queue_term(session->ioctl_queue);
1249                 _mali_osk_free(session);
1250                 MALI_ERROR(_MALI_OSK_ERR_NOMEM);
1251         }
1252
1253         /* Create soft system. */
1254         session->soft_job_system = mali_soft_job_system_create(session);
1255         if (NULL == session->soft_job_system) {
1256                 mali_memory_session_end(session);
1257                 mali_mmu_pagedir_free(session->page_directory);
1258                 _mali_osk_notification_queue_term(session->ioctl_queue);
1259                 _mali_osk_free(session);
1260                 MALI_ERROR(_MALI_OSK_ERR_NOMEM);
1261         }
1262
1263         /* Create timeline system. */
1264         session->timeline_system = mali_timeline_system_create(session);
1265         if (NULL == session->timeline_system) {
1266                 mali_soft_job_system_destroy(session->soft_job_system);
1267                 mali_memory_session_end(session);
1268                 mali_mmu_pagedir_free(session->page_directory);
1269                 _mali_osk_notification_queue_term(session->ioctl_queue);
1270                 _mali_osk_free(session);
1271                 MALI_ERROR(_MALI_OSK_ERR_NOMEM);
1272         }
1273
1274 #if defined(CONFIG_MALI400_POWER_PERFORMANCE_POLICY)
1275         if ( _MALI_OSK_ERR_OK != _mali_osk_atomic_init(&session->number_of_window_jobs, 0)) {
1276                 MALI_DEBUG_PRINT_ERROR(("Initialization of atomic number_of_window_jobs failed.\n"));
1277                 mali_timeline_system_destroy(session->timeline_system);
1278                 mali_soft_job_system_destroy(session->soft_job_system);
1279                 mali_memory_session_end(session);
1280                 mali_mmu_pagedir_free(session->page_directory);
1281                 _mali_osk_notification_queue_term(session->ioctl_queue);
1282                 _mali_osk_free(session);
1283                 return _MALI_OSK_ERR_FAULT;
1284         }
1285 #endif
1286
1287         session->use_high_priority_job_queue = MALI_FALSE;
1288
1289         /* Initialize list of PP jobs on this session. */
1290         _MALI_OSK_INIT_LIST_HEAD(&session->pp_job_list);
1291
1292         /* Initialize the pp_job_fb_lookup_list array used to quickly lookup jobs from a given frame builder */
1293         for (i = 0; i < MALI_PP_JOB_FB_LOOKUP_LIST_SIZE; ++i) {
1294                 _MALI_OSK_INIT_LIST_HEAD(&session->pp_job_fb_lookup_list[i]);
1295         }
1296
1297         *context = (void*)session;
1298
1299         /* Add session to the list of all sessions. */
1300         mali_session_add(session);
1301
1302         MALI_DEBUG_PRINT(2, ("Session started\n"));
1303         MALI_SUCCESS;
1304 }
1305
1306 _mali_osk_errcode_t _mali_ukk_close(void **context)
1307 {
1308         struct mali_session_data *session;
1309         MALI_CHECK_NON_NULL(context, _MALI_OSK_ERR_INVALID_ARGS);
1310         session = (struct mali_session_data *)*context;
1311
1312         MALI_DEBUG_PRINT(3, ("Session ending\n"));
1313
1314         MALI_DEBUG_ASSERT_POINTER(session->soft_job_system);
1315         MALI_DEBUG_ASSERT_POINTER(session->timeline_system);
1316
1317         /* Remove session from list of all sessions. */
1318         mali_session_remove(session);
1319
1320         /* This flag is used to prevent queueing of jobs due to activation. */
1321         session->is_aborting = MALI_TRUE;
1322
1323         /* Stop the soft job timer. */
1324         mali_timeline_system_stop_timer(session->timeline_system);
1325
1326         /* Abort queued and running GP and PP jobs. */
1327         mali_gp_scheduler_abort_session(session);
1328         mali_pp_scheduler_abort_session(session);
1329
1330         /* Abort the soft job system. */
1331         mali_soft_job_system_abort(session->soft_job_system);
1332
1333         /* Force execution of all pending bottom half processing for GP and PP. */
1334         _mali_osk_wq_flush();
1335
1336         /* The session PP list should now be empty. */
1337         MALI_DEBUG_ASSERT(_mali_osk_list_empty(&session->pp_job_list));
1338
1339         /* At this point the GP and PP scheduler no longer has any jobs queued or running from this
1340          * session, and all soft jobs in the soft job system has been destroyed. */
1341
1342         /* Any trackers left in the timeline system are directly or indirectly waiting on external
1343          * sync fences.  Cancel all sync fence waiters to trigger activation of all remaining
1344          * trackers.  This call will sleep until all timelines are empty. */
1345         mali_timeline_system_abort(session->timeline_system);
1346
1347         /* Flush pending work.
1348          * Needed to make sure all bottom half processing related to this
1349          * session has been completed, before we free internal data structures.
1350          */
1351         _mali_osk_wq_flush();
1352
1353         /* Destroy timeline system. */
1354         mali_timeline_system_destroy(session->timeline_system);
1355         session->timeline_system = NULL;
1356
1357         /* Destroy soft system. */
1358         mali_soft_job_system_destroy(session->soft_job_system);
1359         session->soft_job_system = NULL;
1360
1361         MALI_DEBUG_CODE( {
1362                 /* Check that the pp_job_fb_lookup_list array is empty. */
1363                 u32 i;
1364                 for (i = 0; i < MALI_PP_JOB_FB_LOOKUP_LIST_SIZE; ++i)
1365                 {
1366                         MALI_DEBUG_ASSERT(_mali_osk_list_empty(&session->pp_job_fb_lookup_list[i]));
1367                 }
1368         });
1369
1370         /* Free remaining memory allocated to this session */
1371         mali_memory_session_end(session);
1372
1373 #if defined(CONFIG_MALI400_POWER_PERFORMANCE_POLICY)
1374         _mali_osk_atomic_term(&session->number_of_window_jobs);
1375 #endif
1376
1377         /* Free session data structures */
1378         mali_mmu_pagedir_free(session->page_directory);
1379         _mali_osk_notification_queue_term(session->ioctl_queue);
1380         _mali_osk_free(session);
1381
1382         *context = NULL;
1383
1384         MALI_DEBUG_PRINT(2, ("Session has ended\n"));
1385
1386         MALI_SUCCESS;
1387 }
1388
1389 #if MALI_STATE_TRACKING
1390 u32 _mali_kernel_core_dump_state(char* buf, u32 size)
1391 {
1392         int n = 0; /* Number of bytes written to buf */
1393
1394         n += mali_gp_scheduler_dump_state(buf + n, size - n);
1395         n += mali_pp_scheduler_dump_state(buf + n, size - n);
1396
1397         return n;
1398 }
1399 #endif