drivers/gator/gator_events_mali_midgard_hw.c

   1 /**
   2  * Copyright (C) ARM Limited 2012-2014. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 #include "gator.h"
  11
  12 #include <linux/module.h>
  13 #include <linux/time.h>
  14 #include <linux/math64.h>
  15 #include <linux/slab.h>
  16 #include <linux/io.h>
  17
  18 /* Mali Midgard DDK includes */
  19 #if defined(MALI_SIMPLE_API)
  20 /* Header with wrapper functions to kbase structures and functions */
  21 #include "mali/mali_kbase_gator_api.h"
  22 #elif defined(MALI_DIR_MIDGARD)
  23 /* New DDK Directory structure with kernel/drivers/gpu/arm/midgard */
  24 #include "mali_linux_trace.h"
  25 #include "mali_kbase.h"
  26 #include "mali_kbase_mem_linux.h"
  27 #else
  28 /* Old DDK Directory structure with kernel/drivers/gpu/arm/t6xx */
  29 #include "linux/mali_linux_trace.h"
  30 #include "kbase/src/common/mali_kbase.h"
  31 #include "kbase/src/linux/mali_kbase_mem_linux.h"
  32 #endif
  33
  34 /* If API version is not specified then assume API version 1. */
  35 #ifndef MALI_DDK_GATOR_API_VERSION
  36 #define MALI_DDK_GATOR_API_VERSION 1
  37 #endif
  38
  39 #if (MALI_DDK_GATOR_API_VERSION != 1) && (MALI_DDK_GATOR_API_VERSION != 2) && (MALI_DDK_GATOR_API_VERSION != 3)
  40 #error MALI_DDK_GATOR_API_VERSION is invalid (must be 1 for r1/r2 DDK, or 2 for r3/r4 DDK, or 3 for r5 and later DDK).
  41 #endif
  42
  43 #include "gator_events_mali_common.h"
  44
  45 /*
  46  * Mali-Midgard
  47  */
  48 #if MALI_DDK_GATOR_API_VERSION == 3
  49 static uint32_t (*kbase_gator_instr_hwcnt_dump_irq_symbol)(struct kbase_gator_hwcnt_handles *);
  50 static uint32_t (*kbase_gator_instr_hwcnt_dump_complete_symbol)(struct kbase_gator_hwcnt_handles *, uint32_t *const);
  51 static struct kbase_gator_hwcnt_handles *(*kbase_gator_hwcnt_init_symbol)(struct kbase_gator_hwcnt_info *);
  52 static void (*kbase_gator_hwcnt_term_symbol)(struct kbase_gator_hwcnt_info *, struct kbase_gator_hwcnt_handles *);
  53
  54 #else
  55 static struct kbase_device *(*kbase_find_device_symbol)(int);
  56 static struct kbase_context *(*kbase_create_context_symbol)(struct kbase_device *);
  57 static void (*kbase_destroy_context_symbol)(struct kbase_context *);
  58
  59 #if MALI_DDK_GATOR_API_VERSION == 1
  60 static void *(*kbase_va_alloc_symbol)(struct kbase_context *, u32);
  61 static void (*kbase_va_free_symbol)(struct kbase_context *, void *);
  62 #elif MALI_DDK_GATOR_API_VERSION == 2
  63 static void *(*kbase_va_alloc_symbol)(struct kbase_context *, u32, struct kbase_hwc_dma_mapping *);
  64 static void (*kbase_va_free_symbol)(struct kbase_context *, struct kbase_hwc_dma_mapping *);
  65 #endif
  66
  67 static mali_error (*kbase_instr_hwcnt_enable_symbol)(struct kbase_context *, struct kbase_uk_hwcnt_setup *);
  68 static mali_error (*kbase_instr_hwcnt_disable_symbol)(struct kbase_context *);
  69 static mali_error (*kbase_instr_hwcnt_clear_symbol)(struct kbase_context *);
  70 static mali_error (*kbase_instr_hwcnt_dump_irq_symbol)(struct kbase_context *);
  71 static mali_bool (*kbase_instr_hwcnt_dump_complete_symbol)(struct kbase_context *, mali_bool *);
  72
  73 static long shader_present_low;
  74 #endif
  75
  76 /** The interval between reads, in ns.
  77  *
  78  * Earlier we introduced a 'hold off for 1ms after last read' to
  79  * resolve MIDBASE-2178 and MALINE-724. However, the 1ms hold off is
  80  * too long if no context switches occur as there is a race between
  81  * this value and the tick of the read clock in gator which is also
  82  * 1ms. If we 'miss' the current read, the counter values are
  83  * effectively 'spread' over 2ms and the values seen are half what
  84  * they should be (since Streamline averages over sample time). In the
  85  * presence of context switches this spread can vary and markedly
  86  * affect the counters. Currently there is no 'proper' solution to
  87  * this, but empirically we have found that reducing the minimum read
  88  * interval to 950us causes the counts to be much more stable.
  89  */
  90 static const int READ_INTERVAL_NSEC = 950000;
  91
  92 #if GATOR_TEST
  93 #include "gator_events_mali_midgard_hw_test.c"
  94 #endif
  95
  96 #if MALI_DDK_GATOR_API_VERSION != 3
  97 /* Blocks for HW counters */
  98 enum {
  99         JM_BLOCK = 0,
 100         TILER_BLOCK,
 101         SHADER_BLOCK,
 102         MMU_BLOCK
 103 };
 104 #endif
 105
 106 static const char *mali_name;
 107
 108 /* Counters for Mali-Midgard:
 109  *
 110  *    For HW counters we need strings to create /dev/gator/events files.
 111  *    Enums are not needed because the position of the HW name in the array is the same
 112  *    of the corresponding value in the received block of memory.
 113  *    HW counters are requested by calculating a bitmask, passed then to the driver.
 114  *    Every millisecond a HW counters dump is requested, and if the previous has been completed they are read.
 115  */
 116
 117 /* Hardware Counters */
 118 #if MALI_DDK_GATOR_API_VERSION == 3
 119
 120 static const char *const *hardware_counter_names;
 121 static int number_of_hardware_counters;
 122
 123 #else
 124
 125 static const char *const hardware_counter_names[] = {
 126         /* Job Manager */
 127         "",
 128         "",
 129         "",
 130         "",
 131         "MESSAGES_SENT",
 132         "MESSAGES_RECEIVED",
 133         "GPU_ACTIVE",           /* 6 */
 134         "IRQ_ACTIVE",
 135         "JS0_JOBS",
 136         "JS0_TASKS",
 137         "JS0_ACTIVE",
 138         "",
 139         "JS0_WAIT_READ",
 140         "JS0_WAIT_ISSUE",
 141         "JS0_WAIT_DEPEND",
 142         "JS0_WAIT_FINISH",
 143         "JS1_JOBS",
 144         "JS1_TASKS",
 145         "JS1_ACTIVE",
 146         "",
 147         "JS1_WAIT_READ",
 148         "JS1_WAIT_ISSUE",
 149         "JS1_WAIT_DEPEND",
 150         "JS1_WAIT_FINISH",
 151         "JS2_JOBS",
 152         "JS2_TASKS",
 153         "JS2_ACTIVE",
 154         "",
 155         "JS2_WAIT_READ",
 156         "JS2_WAIT_ISSUE",
 157         "JS2_WAIT_DEPEND",
 158         "JS2_WAIT_FINISH",
 159         "JS3_JOBS",
 160         "JS3_TASKS",
 161         "JS3_ACTIVE",
 162         "",
 163         "JS3_WAIT_READ",
 164         "JS3_WAIT_ISSUE",
 165         "JS3_WAIT_DEPEND",
 166         "JS3_WAIT_FINISH",
 167         "JS4_JOBS",
 168         "JS4_TASKS",
 169         "JS4_ACTIVE",
 170         "",
 171         "JS4_WAIT_READ",
 172         "JS4_WAIT_ISSUE",
 173         "JS4_WAIT_DEPEND",
 174         "JS4_WAIT_FINISH",
 175         "JS5_JOBS",
 176         "JS5_TASKS",
 177         "JS5_ACTIVE",
 178         "",
 179         "JS5_WAIT_READ",
 180         "JS5_WAIT_ISSUE",
 181         "JS5_WAIT_DEPEND",
 182         "JS5_WAIT_FINISH",
 183         "JS6_JOBS",
 184         "JS6_TASKS",
 185         "JS6_ACTIVE",
 186         "",
 187         "JS6_WAIT_READ",
 188         "JS6_WAIT_ISSUE",
 189         "JS6_WAIT_DEPEND",
 190         "JS6_WAIT_FINISH",
 191
 192         /*Tiler */
 193         "",
 194         "",
 195         "",
 196         "JOBS_PROCESSED",
 197         "TRIANGLES",
 198         "QUADS",
 199         "POLYGONS",
 200         "POINTS",
 201         "LINES",
 202         "VCACHE_HIT",
 203         "VCACHE_MISS",
 204         "FRONT_FACING",
 205         "BACK_FACING",
 206         "PRIM_VISIBLE",
 207         "PRIM_CULLED",
 208         "PRIM_CLIPPED",
 209         "LEVEL0",
 210         "LEVEL1",
 211         "LEVEL2",
 212         "LEVEL3",
 213         "LEVEL4",
 214         "LEVEL5",
 215         "LEVEL6",
 216         "LEVEL7",
 217         "COMMAND_1",
 218         "COMMAND_2",
 219         "COMMAND_3",
 220         "COMMAND_4",
 221         "COMMAND_4_7",
 222         "COMMAND_8_15",
 223         "COMMAND_16_63",
 224         "COMMAND_64",
 225         "COMPRESS_IN",
 226         "COMPRESS_OUT",
 227         "COMPRESS_FLUSH",
 228         "TIMESTAMPS",
 229         "PCACHE_HIT",
 230         "PCACHE_MISS",
 231         "PCACHE_LINE",
 232         "PCACHE_STALL",
 233         "WRBUF_HIT",
 234         "WRBUF_MISS",
 235         "WRBUF_LINE",
 236         "WRBUF_PARTIAL",
 237         "WRBUF_STALL",
 238         "ACTIVE",
 239         "LOADING_DESC",
 240         "INDEX_WAIT",
 241         "INDEX_RANGE_WAIT",
 242         "VERTEX_WAIT",
 243         "PCACHE_WAIT",
 244         "WRBUF_WAIT",
 245         "BUS_READ",
 246         "BUS_WRITE",
 247         "",
 248         "",
 249         "",
 250         "",
 251         "",
 252         "UTLB_STALL",
 253         "UTLB_REPLAY_MISS",
 254         "UTLB_REPLAY_FULL",
 255         "UTLB_NEW_MISS",
 256         "UTLB_HIT",
 257
 258         /* Shader Core */
 259         "",
 260         "",
 261         "",
 262         "SHADER_CORE_ACTIVE",
 263         "FRAG_ACTIVE",
 264         "FRAG_PRIMATIVES",
 265         "FRAG_PRIMATIVES_DROPPED",
 266         "FRAG_CYCLE_DESC",
 267         "FRAG_CYCLES_PLR",
 268         "FRAG_CYCLES_VERT",
 269         "FRAG_CYCLES_TRISETUP",
 270         "FRAG_CYCLES_RAST",
 271         "FRAG_THREADS",
 272         "FRAG_DUMMY_THREADS",
 273         "FRAG_QUADS_RAST",
 274         "FRAG_QUADS_EZS_TEST",
 275         "FRAG_QUADS_EZS_KILLED",
 276         "FRAG_QUADS_LZS_TEST",
 277         "FRAG_QUADS_LZS_KILLED",
 278         "FRAG_CYCLE_NO_TILE",
 279         "FRAG_NUM_TILES",
 280         "FRAG_TRANS_ELIM",
 281         "COMPUTE_ACTIVE",
 282         "COMPUTE_TASKS",
 283         "COMPUTE_THREADS",
 284         "COMPUTE_CYCLES_DESC",
 285         "TRIPIPE_ACTIVE",
 286         "ARITH_WORDS",
 287         "ARITH_CYCLES_REG",
 288         "ARITH_CYCLES_L0",
 289         "ARITH_FRAG_DEPEND",
 290         "LS_WORDS",
 291         "LS_ISSUES",
 292         "LS_RESTARTS",
 293         "LS_REISSUES_MISS",
 294         "LS_REISSUES_VD",
 295         "LS_REISSUE_ATTRIB_MISS",
 296         "LS_NO_WB",
 297         "TEX_WORDS",
 298         "TEX_BUBBLES",
 299         "TEX_WORDS_L0",
 300         "TEX_WORDS_DESC",
 301         "TEX_THREADS",
 302         "TEX_RECIRC_FMISS",
 303         "TEX_RECIRC_DESC",
 304         "TEX_RECIRC_MULTI",
 305         "TEX_RECIRC_PMISS",
 306         "TEX_RECIRC_CONF",
 307         "LSC_READ_HITS",
 308         "LSC_READ_MISSES",
 309         "LSC_WRITE_HITS",
 310         "LSC_WRITE_MISSES",
 311         "LSC_ATOMIC_HITS",
 312         "LSC_ATOMIC_MISSES",
 313         "LSC_LINE_FETCHES",
 314         "LSC_DIRTY_LINE",
 315         "LSC_SNOOPS",
 316         "AXI_TLB_STALL",
 317         "AXI_TLB_MIESS",
 318         "AXI_TLB_TRANSACTION",
 319         "LS_TLB_MISS",
 320         "LS_TLB_HIT",
 321         "AXI_BEATS_READ",
 322         "AXI_BEATS_WRITTEN",
 323
 324         /*L2 and MMU */
 325         "",
 326         "",
 327         "",
 328         "",
 329         "MMU_HIT",
 330         "MMU_NEW_MISS",
 331         "MMU_REPLAY_FULL",
 332         "MMU_REPLAY_MISS",
 333         "MMU_TABLE_WALK",
 334         "",
 335         "",
 336         "",
 337         "",
 338         "",
 339         "",
 340         "",
 341         "UTLB_HIT",
 342         "UTLB_NEW_MISS",
 343         "UTLB_REPLAY_FULL",
 344         "UTLB_REPLAY_MISS",
 345         "UTLB_STALL",
 346         "",
 347         "",
 348         "",
 349         "",
 350         "",
 351         "",
 352         "",
 353         "",
 354         "",
 355         "L2_WRITE_BEATS",
 356         "L2_READ_BEATS",
 357         "L2_ANY_LOOKUP",
 358         "L2_READ_LOOKUP",
 359         "L2_SREAD_LOOKUP",
 360         "L2_READ_REPLAY",
 361         "L2_READ_SNOOP",
 362         "L2_READ_HIT",
 363         "L2_CLEAN_MISS",
 364         "L2_WRITE_LOOKUP",
 365         "L2_SWRITE_LOOKUP",
 366         "L2_WRITE_REPLAY",
 367         "L2_WRITE_SNOOP",
 368         "L2_WRITE_HIT",
 369         "L2_EXT_READ_FULL",
 370         "L2_EXT_READ_HALF",
 371         "L2_EXT_WRITE_FULL",
 372         "L2_EXT_WRITE_HALF",
 373         "L2_EXT_READ",
 374         "L2_EXT_READ_LINE",
 375         "L2_EXT_WRITE",
 376         "L2_EXT_WRITE_LINE",
 377         "L2_EXT_WRITE_SMALL",
 378         "L2_EXT_BARRIER",
 379         "L2_EXT_AR_STALL",
 380         "L2_EXT_R_BUF_FULL",
 381         "L2_EXT_RD_BUF_FULL",
 382         "L2_EXT_R_RAW",
 383         "L2_EXT_W_STALL",
 384         "L2_EXT_W_BUF_FULL",
 385         "L2_EXT_R_W_HAZARD",
 386         "L2_TAG_HAZARD",
 387         "L2_SNOOP_FULL",
 388         "L2_REPLAY_FULL"
 389 };
 390
 391 static const int number_of_hardware_counters = ARRAY_SIZE(hardware_counter_names);
 392
 393 #endif
 394
 395 #define GET_HW_BLOCK(c) (((c) >> 6) & 0x3)
 396 #define GET_COUNTER_OFFSET(c) ((c) & 0x3f)
 397
 398 #if MALI_DDK_GATOR_API_VERSION == 3
 399 /* Opaque handles for kbase_context and kbase_hwc_dma_mapping */
 400 static struct kbase_gator_hwcnt_handles *handles;
 401
 402 /* Information about hardware counters */
 403 static struct kbase_gator_hwcnt_info *in_out_info;
 404
 405 #else
 406 /* Memory to dump hardware counters into */
 407 static void *kernel_dump_buffer;
 408
 409 #if MALI_DDK_GATOR_API_VERSION == 2
 410 /* DMA state used to manage lifetime of the buffer */
 411 struct kbase_hwc_dma_mapping kernel_dump_buffer_handle;
 412 #endif
 413
 414 /* kbase context and device */
 415 static struct kbase_context *kbcontext;
 416 static struct kbase_device *kbdevice;
 417
 418 /*
 419  * The following function has no external prototype in older DDK
 420  * revisions. When the DDK is updated then this should be removed.
 421  */
 422 struct kbase_device *kbase_find_device(int minor);
 423 #endif
 424
 425 static volatile bool kbase_device_busy;
 426 static unsigned int num_hardware_counters_enabled;
 427
 428 /* gatorfs variables for counter enable state */
 429 static struct mali_counter *counters;
 430
 431 /* An array used to return the data we recorded as key,value pairs */
 432 static int *counter_dump;
 433
 434 extern struct mali_counter mali_activity[3];
 435
 436 static const char *const mali_activity_names[] = {
 437         "fragment",
 438         "vertex",
 439         "opencl",
 440 };
 441
 442 #define SYMBOL_GET(FUNCTION, ERROR_COUNT) \
 443         do { \
 444                 if (FUNCTION ## _symbol) { \
 445                         pr_err("gator: mali " #FUNCTION " symbol was already registered\n"); \
 446                         (ERROR_COUNT)++; \
 447                 } else { \
 448                         FUNCTION ## _symbol = symbol_get(FUNCTION); \
 449                         if (!FUNCTION ## _symbol) { \
 450                                 pr_err("gator: mali online " #FUNCTION " symbol not found\n"); \
 451                                 (ERROR_COUNT)++; \
 452                         } \
 453                 } \
 454         } while (0)
 455
 456 #define SYMBOL_CLEANUP(FUNCTION) \
 457         do { \
 458                 if (FUNCTION ## _symbol) { \
 459                         symbol_put(FUNCTION); \
 460                         FUNCTION ## _symbol = NULL; \
 461                 } \
 462         } while (0)
 463
 464 /**
 465  * Execute symbol_get for all the Mali symbols and check for success.
 466  * @return the number of symbols not loaded.
 467  */
 468 static int init_symbols(void)
 469 {
 470         int error_count = 0;
 471 #if MALI_DDK_GATOR_API_VERSION == 3
 472         SYMBOL_GET(kbase_gator_instr_hwcnt_dump_irq, error_count);
 473         SYMBOL_GET(kbase_gator_instr_hwcnt_dump_complete, error_count);
 474         SYMBOL_GET(kbase_gator_hwcnt_init, error_count);
 475         SYMBOL_GET(kbase_gator_hwcnt_term, error_count);
 476 #else
 477         SYMBOL_GET(kbase_find_device, error_count);
 478         SYMBOL_GET(kbase_create_context, error_count);
 479         SYMBOL_GET(kbase_va_alloc, error_count);
 480         SYMBOL_GET(kbase_instr_hwcnt_enable, error_count);
 481         SYMBOL_GET(kbase_instr_hwcnt_clear, error_count);
 482         SYMBOL_GET(kbase_instr_hwcnt_dump_irq, error_count);
 483         SYMBOL_GET(kbase_instr_hwcnt_dump_complete, error_count);
 484         SYMBOL_GET(kbase_instr_hwcnt_disable, error_count);
 485         SYMBOL_GET(kbase_va_free, error_count);
 486         SYMBOL_GET(kbase_destroy_context, error_count);
 487 #endif
 488
 489         return error_count;
 490 }
 491
 492 /**
 493  * Execute symbol_put for all the registered Mali symbols.
 494  */
 495 static void clean_symbols(void)
 496 {
 497 #if MALI_DDK_GATOR_API_VERSION == 3
 498         SYMBOL_CLEANUP(kbase_gator_instr_hwcnt_dump_irq);
 499         SYMBOL_CLEANUP(kbase_gator_instr_hwcnt_dump_complete);
 500         SYMBOL_CLEANUP(kbase_gator_hwcnt_init);
 501         SYMBOL_CLEANUP(kbase_gator_hwcnt_term);
 502 #else
 503         SYMBOL_CLEANUP(kbase_find_device);
 504         SYMBOL_CLEANUP(kbase_create_context);
 505         SYMBOL_CLEANUP(kbase_va_alloc);
 506         SYMBOL_CLEANUP(kbase_instr_hwcnt_enable);
 507         SYMBOL_CLEANUP(kbase_instr_hwcnt_clear);
 508         SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_irq);
 509         SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_complete);
 510         SYMBOL_CLEANUP(kbase_instr_hwcnt_disable);
 511         SYMBOL_CLEANUP(kbase_va_free);
 512         SYMBOL_CLEANUP(kbase_destroy_context);
 513 #endif
 514 }
 515
 516 /**
 517  * Determines whether a read should take place
 518  * @param current_time The current time, obtained from getnstimeofday()
 519  * @param prev_time_s The number of seconds at the previous read attempt.
 520  * @param next_read_time_ns The time (in ns) when the next read should be allowed.
 521  *
 522  * Note that this function has been separated out here to allow it to be tested.
 523  */
 524 static int is_read_scheduled(const struct timespec *current_time, u32 *prev_time_s, s32 *next_read_time_ns)
 525 {
 526         /* If the current ns count rolls over a second, roll the next read time too. */
 527         if (current_time->tv_sec != *prev_time_s)
 528                 *next_read_time_ns = *next_read_time_ns - NSEC_PER_SEC;
 529
 530         /* Abort the read if the next read time has not arrived. */
 531         if (current_time->tv_nsec < *next_read_time_ns)
 532                 return 0;
 533
 534         /* Set the next read some fixed time after this one, and update the read timestamp. */
 535         *next_read_time_ns = current_time->tv_nsec + READ_INTERVAL_NSEC;
 536
 537         *prev_time_s = current_time->tv_sec;
 538         return 1;
 539 }
 540
 541 static int start(void)
 542 {
 543 #if MALI_DDK_GATOR_API_VERSION != 3
 544         struct kbase_uk_hwcnt_setup setup;
 545         unsigned long long shadersPresent = 0;
 546         u16 bitmask[] = { 0, 0, 0, 0 };
 547         mali_error err;
 548 #endif
 549         int cnt;
 550
 551 #if MALI_DDK_GATOR_API_VERSION == 3
 552         /* Setup HW counters */
 553         num_hardware_counters_enabled = 0;
 554
 555         /* Declare and initialise kbase_gator_hwcnt_info structure */
 556         in_out_info = kmalloc(sizeof(*in_out_info), GFP_KERNEL);
 557         for (cnt = 0; cnt < ARRAY_SIZE(in_out_info->bitmask); cnt++)
 558                 in_out_info->bitmask[cnt] = 0;
 559
 560         /* Calculate enable bitmasks based on counters_enabled array */
 561         for (cnt = 0; cnt < number_of_hardware_counters; cnt++) {
 562                 if (counters[cnt].enabled) {
 563                         int block = GET_HW_BLOCK(cnt);
 564                         int enable_bit = GET_COUNTER_OFFSET(cnt) / 4;
 565
 566                         in_out_info->bitmask[block] |= (1 << enable_bit);
 567                         pr_debug("gator: Mali-Midgard: hardware counter %s selected [%d]\n", hardware_counter_names[cnt], cnt);
 568                         num_hardware_counters_enabled++;
 569                 }
 570         }
 571
 572         /* Create a kbase context for HW counters */
 573         if (num_hardware_counters_enabled > 0) {
 574                 if (init_symbols() > 0) {
 575                         clean_symbols();
 576                         /* No Mali driver code entrypoints found - not a fault. */
 577                         return 0;
 578                 }
 579
 580                 handles = kbase_gator_hwcnt_init_symbol(in_out_info);
 581
 582                 if (handles == NULL)
 583                         goto out;
 584
 585                 kbase_device_busy = false;
 586         }
 587
 588         return 0;
 589 #else
 590         /* Setup HW counters */
 591         num_hardware_counters_enabled = 0;
 592
 593         /* Calculate enable bitmasks based on counters_enabled array */
 594         for (cnt = 0; cnt < number_of_hardware_counters; cnt++) {
 595                 const struct mali_counter *counter = &counters[cnt];
 596
 597                 if (counter->enabled) {
 598                         int block = GET_HW_BLOCK(cnt);
 599                         int enable_bit = GET_COUNTER_OFFSET(cnt) / 4;
 600
 601                         bitmask[block] |= (1 << enable_bit);
 602                         pr_debug("gator: Mali-Midgard: hardware counter %s selected [%d]\n", hardware_counter_names[cnt], cnt);
 603                         num_hardware_counters_enabled++;
 604                 }
 605         }
 606
 607         /* Create a kbase context for HW counters */
 608         if (num_hardware_counters_enabled > 0) {
 609                 if (init_symbols() > 0) {
 610                         clean_symbols();
 611                         /* No Mali driver code entrypoints found - not a fault. */
 612                         return 0;
 613                 }
 614
 615                 kbdevice = kbase_find_device_symbol(-1);
 616
 617                 /* If we already got a context, fail */
 618                 if (kbcontext) {
 619                         pr_debug("gator: Mali-Midgard: error context already present\n");
 620                         goto out;
 621                 }
 622
 623                 /* kbcontext will only be valid after all the Mali symbols are loaded successfully */
 624                 kbcontext = kbase_create_context_symbol(kbdevice);
 625                 if (!kbcontext) {
 626                         pr_debug("gator: Mali-Midgard: error creating kbase context\n");
 627                         goto out;
 628                 }
 629
 630                 /* See if we can get the number of shader cores */
 631                 shadersPresent = kbdevice->shader_present_bitmap;
 632                 shader_present_low = (unsigned long)shadersPresent;
 633
 634                 /*
 635                  * The amount of memory needed to store the dump (bytes)
 636                  * DUMP_SIZE = number of core groups
 637                  *             * number of blocks (always 8 for midgard)
 638                  *             * number of counters per block (always 64 for midgard)
 639                  *             * number of bytes per counter (always 4 in midgard)
 640                  * For a Mali-Midgard with a single core group = 1 * 8 * 64 * 4 = 2048
 641                  * For a Mali-Midgard with a dual core group   = 2 * 8 * 64 * 4 = 4096
 642                  */
 643 #if MALI_DDK_GATOR_API_VERSION == 1
 644                 kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096);
 645 #elif MALI_DDK_GATOR_API_VERSION == 2
 646                 kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096, &kernel_dump_buffer_handle);
 647 #endif
 648                 if (!kernel_dump_buffer) {
 649                         pr_debug("gator: Mali-Midgard: error trying to allocate va\n");
 650                         goto destroy_context;
 651                 }
 652
 653                 setup.dump_buffer = (uintptr_t)kernel_dump_buffer;
 654                 setup.jm_bm = bitmask[JM_BLOCK];
 655                 setup.tiler_bm = bitmask[TILER_BLOCK];
 656                 setup.shader_bm = bitmask[SHADER_BLOCK];
 657                 setup.mmu_l2_bm = bitmask[MMU_BLOCK];
 658                 /* These counters do not exist on Mali-T60x */
 659                 setup.l3_cache_bm = 0;
 660
 661                 /* Use kbase API to enable hardware counters and provide dump buffer */
 662                 err = kbase_instr_hwcnt_enable_symbol(kbcontext, &setup);
 663                 if (err != MALI_ERROR_NONE) {
 664                         pr_debug("gator: Mali-Midgard: can't setup hardware counters\n");
 665                         goto free_buffer;
 666                 }
 667                 pr_debug("gator: Mali-Midgard: hardware counters enabled\n");
 668                 kbase_instr_hwcnt_clear_symbol(kbcontext);
 669                 pr_debug("gator: Mali-Midgard: hardware counters cleared\n");
 670
 671                 kbase_device_busy = false;
 672         }
 673
 674         return 0;
 675
 676 free_buffer:
 677 #if MALI_DDK_GATOR_API_VERSION == 1
 678         kbase_va_free_symbol(kbcontext, kernel_dump_buffer);
 679 #elif MALI_DDK_GATOR_API_VERSION == 2
 680         kbase_va_free_symbol(kbcontext, &kernel_dump_buffer_handle);
 681 #endif
 682
 683 destroy_context:
 684         kbase_destroy_context_symbol(kbcontext);
 685 #endif
 686
 687 out:
 688         clean_symbols();
 689         return -1;
 690 }
 691
 692 static void stop(void)
 693 {
 694         unsigned int cnt;
 695 #if MALI_DDK_GATOR_API_VERSION == 3
 696         struct kbase_gator_hwcnt_handles *temp_hand;
 697 #else
 698         struct kbase_context *temp_kbcontext;
 699 #endif
 700
 701         pr_debug("gator: Mali-Midgard: stop\n");
 702
 703         /* Set all counters as disabled */
 704         for (cnt = 0; cnt < number_of_hardware_counters; cnt++)
 705                 counters[cnt].enabled = 0;
 706
 707         /* Destroy the context for HW counters */
 708 #if MALI_DDK_GATOR_API_VERSION == 3
 709         if (num_hardware_counters_enabled > 0 && handles != NULL) {
 710                 /*
 711                  * Set the global variable to NULL before destroying it, because
 712                  * other function will check this before using it.
 713                  */
 714                 temp_hand = handles;
 715                 handles = NULL;
 716
 717                 kbase_gator_hwcnt_term_symbol(in_out_info, temp_hand);
 718
 719                 kfree(in_out_info);
 720
 721 #else
 722         if (num_hardware_counters_enabled > 0 && kbcontext != NULL) {
 723                 /*
 724                  * Set the global variable to NULL before destroying it, because
 725                  * other function will check this before using it.
 726                  */
 727                 temp_kbcontext = kbcontext;
 728                 kbcontext = NULL;
 729
 730                 kbase_instr_hwcnt_disable_symbol(temp_kbcontext);
 731
 732 #if MALI_DDK_GATOR_API_VERSION == 1
 733                 kbase_va_free_symbol(temp_kbcontext, kernel_dump_buffer);
 734 #elif MALI_DDK_GATOR_API_VERSION == 2
 735                 kbase_va_free_symbol(temp_kbcontext, &kernel_dump_buffer_handle);
 736 #endif
 737
 738                 kbase_destroy_context_symbol(temp_kbcontext);
 739 #endif
 740
 741                 pr_debug("gator: Mali-Midgard: hardware counters stopped\n");
 742
 743                 clean_symbols();
 744         }
 745 }
 746
 747 static int read_counter(const int cnt, const int len, const struct mali_counter *counter)
 748 {
 749         const int block = GET_HW_BLOCK(cnt);
 750         const int counter_offset = GET_COUNTER_OFFSET(cnt);
 751
 752 #if MALI_DDK_GATOR_API_VERSION == 3
 753         const char *block_base_address = (char *)in_out_info->kernel_dump_buffer;
 754         int i;
 755         int shader_core_count = 0;
 756         u32 value = 0;
 757
 758         for (i = 0; i < in_out_info->nr_hwc_blocks; i++) {
 759                 if (block == in_out_info->hwc_layout[i]) {
 760                         value += *((u32 *)(block_base_address + (0x100 * i)) + counter_offset);
 761                         if (block == SHADER_BLOCK)
 762                                 ++shader_core_count;
 763                 }
 764         }
 765
 766         if (shader_core_count > 1)
 767                 value /= shader_core_count;
 768 #else
 769         const char *block_base_address = (char *)kernel_dump_buffer + vithar_blocks[block];
 770
 771         /* If counter belongs to shader block need to take into account all cores */
 772         if (block == SHADER_BLOCK) {
 773                 int i = 0;
 774                 int shader_core_count = 0;
 775
 776                 value = 0;
 777
 778                 for (i = 0; i < 4; i++) {
 779                         if ((shader_present_low >> i) & 1) {
 780                                 value += *((u32 *)(block_base_address + (0x100 * i)) + counter_offset);
 781                                 shader_core_count++;
 782                         }
 783                 }
 784
 785                 for (i = 0; i < 4; i++) {
 786                         if ((shader_present_low >> (i+4)) & 1) {
 787                                 value += *((u32 *)(block_base_address + (0x100 * i) + 0x800) + counter_offset);
 788                                 shader_core_count++;
 789                         }
 790                 }
 791
 792                 /* Need to total by number of cores to produce an average */
 793                 if (shader_core_count != 0)
 794                         value /= shader_core_count;
 795         } else {
 796                 value = *((u32 *)block_base_address + counter_offset);
 797         }
 798 #endif
 799
 800         counter_dump[len + 0] = counter->key;
 801         counter_dump[len + 1] = value;
 802
 803         return 2;
 804 }
 805
 806 static int read(int **buffer, bool sched_switch)
 807 {
 808         int cnt;
 809         int len = 0;
 810         uint32_t success;
 811
 812         struct timespec current_time;
 813         static u32 prev_time_s;
 814         static s32 next_read_time_ns;
 815
 816         if (!on_primary_core() || sched_switch)
 817                 return 0;
 818
 819         getnstimeofday(&current_time);
 820
 821         /*
 822          * Discard reads unless a respectable time has passed. This
 823          * reduces the load on the GPU without sacrificing accuracy on
 824          * the Streamline display.
 825          */
 826         if (!is_read_scheduled(&current_time, &prev_time_s, &next_read_time_ns))
 827                 return 0;
 828
 829         /*
 830          * Report the HW counters
 831          * Only process hardware counters if at least one of the hardware counters is enabled.
 832          */
 833         if (num_hardware_counters_enabled > 0) {
 834 #if MALI_DDK_GATOR_API_VERSION != 3
 835                 const unsigned int vithar_blocks[] = {
 836                         0x700,  /* VITHAR_JOB_MANAGER,     Block 0 */
 837                         0x400,  /* VITHAR_TILER,           Block 1 */
 838                         0x000,  /* VITHAR_SHADER_CORE,     Block 2 */
 839                         0x500   /* VITHAR_MEMORY_SYSTEM,   Block 3 */
 840                 };
 841 #endif
 842
 843 #if MALI_DDK_GATOR_API_VERSION == 3
 844                 if (!handles)
 845                         return -1;
 846
 847                 /* Mali symbols can be called safely since a kbcontext is valid */
 848                 if (kbase_gator_instr_hwcnt_dump_complete_symbol(handles, &success) == MALI_TRUE) {
 849 #else
 850                 if (!kbcontext)
 851                         return -1;
 852
 853                 /* Mali symbols can be called safely since a kbcontext is valid */
 854                 if (kbase_instr_hwcnt_dump_complete_symbol(kbcontext, &success) == MALI_TRUE) {
 855 #endif
 856                         kbase_device_busy = false;
 857
 858                         if (success == MALI_TRUE) {
 859                                 /* Cycle through hardware counters and accumulate totals */
 860                                 for (cnt = 0; cnt < number_of_hardware_counters; cnt++) {
 861                                         const struct mali_counter *counter = &counters[cnt];
 862
 863                                         if (counter->enabled)
 864                                                 len += read_counter(cnt, len, counter);
 865                                 }
 866                         }
 867                 }
 868
 869                 if (!kbase_device_busy) {
 870                         kbase_device_busy = true;
 871 #if MALI_DDK_GATOR_API_VERSION == 3
 872                         kbase_gator_instr_hwcnt_dump_irq_symbol(handles);
 873 #else
 874                         kbase_instr_hwcnt_dump_irq_symbol(kbcontext);
 875 #endif
 876                 }
 877         }
 878
 879         /* Update the buffer */
 880         if (buffer)
 881                 *buffer = counter_dump;
 882
 883         return len;
 884 }
 885
 886 static int create_files(struct super_block *sb, struct dentry *root)
 887 {
 888         unsigned int event;
 889         /*
 890          * Create the filesystem for all events
 891          */
 892         for (event = 0; event < ARRAY_SIZE(mali_activity); event++) {
 893                 if (gator_mali_create_file_system("Midgard", mali_activity_names[event], sb, root, &mali_activity[event], NULL) != 0)
 894                         return -1;
 895         }
 896
 897         for (event = 0; event < number_of_hardware_counters; event++) {
 898                 if (gator_mali_create_file_system(mali_name, hardware_counter_names[event], sb, root, &counters[event], NULL) != 0)
 899                         return -1;
 900         }
 901
 902         return 0;
 903 }
 904
 905 static void shutdown(void)
 906 {
 907 #if MALI_DDK_GATOR_API_VERSION == 3
 908         void (*kbase_gator_hwcnt_term_names_symbol)(void) = NULL;
 909         int error_count = 0;
 910 #endif
 911
 912         kfree(counters);
 913         kfree(counter_dump);
 914
 915 #if MALI_DDK_GATOR_API_VERSION == 3
 916         SYMBOL_GET(kbase_gator_hwcnt_term_names, error_count);
 917
 918         number_of_hardware_counters = -1;
 919         hardware_counter_names = NULL;
 920         if (kbase_gator_hwcnt_term_names_symbol != NULL) {
 921                 kbase_gator_hwcnt_term_names_symbol();
 922                 pr_err("Released symbols\n");
 923         }
 924
 925         SYMBOL_CLEANUP(kbase_gator_hwcnt_term_names);
 926 #endif
 927 }
 928
 929 static struct gator_interface gator_events_mali_midgard_interface = {
 930         .shutdown = shutdown,
 931         .create_files = create_files,
 932         .start = start,
 933         .stop = stop,
 934         .read = read
 935 };
 936
 937 int gator_events_mali_midgard_hw_init(void)
 938 {
 939 #if MALI_DDK_GATOR_API_VERSION == 3
 940         const char *const *(*kbase_gator_hwcnt_init_names_symbol)(uint32_t *) = NULL;
 941         int error_count = 0;
 942 #endif
 943
 944         pr_debug("gator: Mali-Midgard: sw_counters init\n");
 945
 946 #if GATOR_TEST
 947         test_all_is_read_scheduled();
 948 #endif
 949
 950 #if MALI_DDK_GATOR_API_VERSION == 3
 951         SYMBOL_GET(kbase_gator_hwcnt_init_names, error_count);
 952         if (error_count > 0) {
 953                 SYMBOL_CLEANUP(kbase_gator_hwcnt_init_names);
 954                 return 1;
 955         }
 956
 957         number_of_hardware_counters = -1;
 958         hardware_counter_names = kbase_gator_hwcnt_init_names_symbol(&number_of_hardware_counters);
 959
 960         SYMBOL_CLEANUP(kbase_gator_hwcnt_init_names);
 961
 962         if ((hardware_counter_names == NULL) || (number_of_hardware_counters <= 0)) {
 963                 pr_err("gator: Error reading hardware counters names: got %d names\n", number_of_hardware_counters);
 964                 return -1;
 965         }
 966 #else
 967         mali_name = "Midgard";
 968 #endif
 969
 970         counters = kmalloc(sizeof(*counters)*number_of_hardware_counters, GFP_KERNEL);
 971         counter_dump = kmalloc(sizeof(*counter_dump)*number_of_hardware_counters*2, GFP_KERNEL);
 972
 973         gator_mali_initialise_counters(mali_activity, ARRAY_SIZE(mali_activity));
 974         gator_mali_initialise_counters(counters, number_of_hardware_counters);
 975
 976         return gator_events_install(&gator_events_mali_midgard_interface);
 977 }