drivers/gator/gator_events_mali_midgard_hw.c

   1 /**
   2  * Copyright (C) ARM Limited 2012-2015. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 #include "gator.h"
  11
  12 #include <linux/module.h>
  13 #include <linux/time.h>
  14 #include <linux/math64.h>
  15 #include <linux/slab.h>
  16 #include <linux/io.h>
  17
  18 /* Mali Midgard DDK includes */
  19 #if defined(MALI_SIMPLE_API)
  20 /* Header with wrapper functions to kbase structures and functions */
  21 #include "mali_kbase_gator_api.h"
  22 #elif defined(MALI_DIR_MIDGARD)
  23 /* New DDK Directory structure with kernel/drivers/gpu/arm/midgard */
  24 #include "mali_linux_trace.h"
  25 #include "mali_kbase.h"
  26 #include "mali_kbase_mem_linux.h"
  27 #else
  28 /* Old DDK Directory structure with kernel/drivers/gpu/arm/t6xx */
  29 #include "linux/mali_linux_trace.h"
  30 #include "kbase/src/common/mali_kbase.h"
  31 #include "kbase/src/linux/mali_kbase_mem_linux.h"
  32 #endif
  33
  34 /* If API version is not specified then assume API version 1. */
  35 #ifndef MALI_DDK_GATOR_API_VERSION
  36 #define MALI_DDK_GATOR_API_VERSION 1
  37 #endif
  38
  39 #if (MALI_DDK_GATOR_API_VERSION != 1) && (MALI_DDK_GATOR_API_VERSION != 2) && (MALI_DDK_GATOR_API_VERSION != 3)
  40 #error MALI_DDK_GATOR_API_VERSION is invalid (must be 1 for r1/r2 DDK, or 2 for r3/r4 DDK, or 3 for r5 and later DDK).
  41 #endif
  42
  43 #if !defined(CONFIG_MALI_GATOR_SUPPORT)
  44 #error CONFIG_MALI_GATOR_SUPPORT is required for GPU activity and software counters
  45 #endif
  46
  47 #include "gator_events_mali_common.h"
  48
  49 /*
  50  * Mali-Midgard
  51  */
  52 #if MALI_DDK_GATOR_API_VERSION == 3
  53 static uint32_t (*kbase_gator_instr_hwcnt_dump_irq_symbol)(struct kbase_gator_hwcnt_handles *);
  54 static uint32_t (*kbase_gator_instr_hwcnt_dump_complete_symbol)(struct kbase_gator_hwcnt_handles *, uint32_t *const);
  55 static struct kbase_gator_hwcnt_handles *(*kbase_gator_hwcnt_init_symbol)(struct kbase_gator_hwcnt_info *);
  56 static void (*kbase_gator_hwcnt_term_symbol)(struct kbase_gator_hwcnt_info *, struct kbase_gator_hwcnt_handles *);
  57
  58 #else
  59 static struct kbase_device *(*kbase_find_device_symbol)(int);
  60 static struct kbase_context *(*kbase_create_context_symbol)(struct kbase_device *);
  61 static void (*kbase_destroy_context_symbol)(struct kbase_context *);
  62
  63 #if MALI_DDK_GATOR_API_VERSION == 1
  64 static void *(*kbase_va_alloc_symbol)(struct kbase_context *, u32);
  65 static void (*kbase_va_free_symbol)(struct kbase_context *, void *);
  66 #elif MALI_DDK_GATOR_API_VERSION == 2
  67 static void *(*kbase_va_alloc_symbol)(struct kbase_context *, u32, struct kbase_hwc_dma_mapping *);
  68 static void (*kbase_va_free_symbol)(struct kbase_context *, struct kbase_hwc_dma_mapping *);
  69 #endif
  70
  71 static mali_error (*kbase_instr_hwcnt_enable_symbol)(struct kbase_context *, struct kbase_uk_hwcnt_setup *);
  72 static mali_error (*kbase_instr_hwcnt_disable_symbol)(struct kbase_context *);
  73 static mali_error (*kbase_instr_hwcnt_clear_symbol)(struct kbase_context *);
  74 static mali_error (*kbase_instr_hwcnt_dump_irq_symbol)(struct kbase_context *);
  75 static mali_bool (*kbase_instr_hwcnt_dump_complete_symbol)(struct kbase_context *, mali_bool *);
  76
  77 static long shader_present_low;
  78 #endif
  79
  80 /** The interval between reads, in ns.
  81  *
  82  * Earlier we introduced a 'hold off for 1ms after last read' to
  83  * resolve MIDBASE-2178 and MALINE-724. However, the 1ms hold off is
  84  * too long if no context switches occur as there is a race between
  85  * this value and the tick of the read clock in gator which is also
  86  * 1ms. If we 'miss' the current read, the counter values are
  87  * effectively 'spread' over 2ms and the values seen are half what
  88  * they should be (since Streamline averages over sample time). In the
  89  * presence of context switches this spread can vary and markedly
  90  * affect the counters. Currently there is no 'proper' solution to
  91  * this, but empirically we have found that reducing the minimum read
  92  * interval to 950us causes the counts to be much more stable.
  93  */
  94 static const int READ_INTERVAL_NSEC = 950000;
  95
  96 #if GATOR_TEST
  97 #include "gator_events_mali_midgard_hw_test.c"
  98 #endif
  99
 100 #if MALI_DDK_GATOR_API_VERSION != 3
 101 /* Blocks for HW counters */
 102 enum {
 103         JM_BLOCK = 0,
 104         TILER_BLOCK,
 105         SHADER_BLOCK,
 106         MMU_BLOCK
 107 };
 108 #endif
 109
 110 static const char *mali_name;
 111
 112 /* Counters for Mali-Midgard:
 113  *
 114  *    For HW counters we need strings to create /dev/gator/events files.
 115  *    Enums are not needed because the position of the HW name in the array is the same
 116  *    of the corresponding value in the received block of memory.
 117  *    HW counters are requested by calculating a bitmask, passed then to the driver.
 118  *    Every millisecond a HW counters dump is requested, and if the previous has been completed they are read.
 119  */
 120
 121 /* Hardware Counters */
 122 #if MALI_DDK_GATOR_API_VERSION == 3
 123
 124 static const char *const *hardware_counter_names;
 125 static int number_of_hardware_counters;
 126
 127 #else
 128
 129 static const char *const hardware_counter_names[] = {
 130         /* Job Manager */
 131         "",
 132         "",
 133         "",
 134         "",
 135         "MESSAGES_SENT",
 136         "MESSAGES_RECEIVED",
 137         "GPU_ACTIVE",           /* 6 */
 138         "IRQ_ACTIVE",
 139         "JS0_JOBS",
 140         "JS0_TASKS",
 141         "JS0_ACTIVE",
 142         "",
 143         "JS0_WAIT_READ",
 144         "JS0_WAIT_ISSUE",
 145         "JS0_WAIT_DEPEND",
 146         "JS0_WAIT_FINISH",
 147         "JS1_JOBS",
 148         "JS1_TASKS",
 149         "JS1_ACTIVE",
 150         "",
 151         "JS1_WAIT_READ",
 152         "JS1_WAIT_ISSUE",
 153         "JS1_WAIT_DEPEND",
 154         "JS1_WAIT_FINISH",
 155         "JS2_JOBS",
 156         "JS2_TASKS",
 157         "JS2_ACTIVE",
 158         "",
 159         "JS2_WAIT_READ",
 160         "JS2_WAIT_ISSUE",
 161         "JS2_WAIT_DEPEND",
 162         "JS2_WAIT_FINISH",
 163         "JS3_JOBS",
 164         "JS3_TASKS",
 165         "JS3_ACTIVE",
 166         "",
 167         "JS3_WAIT_READ",
 168         "JS3_WAIT_ISSUE",
 169         "JS3_WAIT_DEPEND",
 170         "JS3_WAIT_FINISH",
 171         "JS4_JOBS",
 172         "JS4_TASKS",
 173         "JS4_ACTIVE",
 174         "",
 175         "JS4_WAIT_READ",
 176         "JS4_WAIT_ISSUE",
 177         "JS4_WAIT_DEPEND",
 178         "JS4_WAIT_FINISH",
 179         "JS5_JOBS",
 180         "JS5_TASKS",
 181         "JS5_ACTIVE",
 182         "",
 183         "JS5_WAIT_READ",
 184         "JS5_WAIT_ISSUE",
 185         "JS5_WAIT_DEPEND",
 186         "JS5_WAIT_FINISH",
 187         "JS6_JOBS",
 188         "JS6_TASKS",
 189         "JS6_ACTIVE",
 190         "",
 191         "JS6_WAIT_READ",
 192         "JS6_WAIT_ISSUE",
 193         "JS6_WAIT_DEPEND",
 194         "JS6_WAIT_FINISH",
 195
 196         /*Tiler */
 197         "",
 198         "",
 199         "",
 200         "JOBS_PROCESSED",
 201         "TRIANGLES",
 202         "QUADS",
 203         "POLYGONS",
 204         "POINTS",
 205         "LINES",
 206         "VCACHE_HIT",
 207         "VCACHE_MISS",
 208         "FRONT_FACING",
 209         "BACK_FACING",
 210         "PRIM_VISIBLE",
 211         "PRIM_CULLED",
 212         "PRIM_CLIPPED",
 213         "LEVEL0",
 214         "LEVEL1",
 215         "LEVEL2",
 216         "LEVEL3",
 217         "LEVEL4",
 218         "LEVEL5",
 219         "LEVEL6",
 220         "LEVEL7",
 221         "COMMAND_1",
 222         "COMMAND_2",
 223         "COMMAND_3",
 224         "COMMAND_4",
 225         "COMMAND_4_7",
 226         "COMMAND_8_15",
 227         "COMMAND_16_63",
 228         "COMMAND_64",
 229         "COMPRESS_IN",
 230         "COMPRESS_OUT",
 231         "COMPRESS_FLUSH",
 232         "TIMESTAMPS",
 233         "PCACHE_HIT",
 234         "PCACHE_MISS",
 235         "PCACHE_LINE",
 236         "PCACHE_STALL",
 237         "WRBUF_HIT",
 238         "WRBUF_MISS",
 239         "WRBUF_LINE",
 240         "WRBUF_PARTIAL",
 241         "WRBUF_STALL",
 242         "ACTIVE",
 243         "LOADING_DESC",
 244         "INDEX_WAIT",
 245         "INDEX_RANGE_WAIT",
 246         "VERTEX_WAIT",
 247         "PCACHE_WAIT",
 248         "WRBUF_WAIT",
 249         "BUS_READ",
 250         "BUS_WRITE",
 251         "",
 252         "",
 253         "",
 254         "",
 255         "",
 256         "UTLB_STALL",
 257         "UTLB_REPLAY_MISS",
 258         "UTLB_REPLAY_FULL",
 259         "UTLB_NEW_MISS",
 260         "UTLB_HIT",
 261
 262         /* Shader Core */
 263         "",
 264         "",
 265         "",
 266         "SHADER_CORE_ACTIVE",
 267         "FRAG_ACTIVE",
 268         "FRAG_PRIMATIVES",
 269         "FRAG_PRIMATIVES_DROPPED",
 270         "FRAG_CYCLE_DESC",
 271         "FRAG_CYCLES_PLR",
 272         "FRAG_CYCLES_VERT",
 273         "FRAG_CYCLES_TRISETUP",
 274         "FRAG_CYCLES_RAST",
 275         "FRAG_THREADS",
 276         "FRAG_DUMMY_THREADS",
 277         "FRAG_QUADS_RAST",
 278         "FRAG_QUADS_EZS_TEST",
 279         "FRAG_QUADS_EZS_KILLED",
 280         "FRAG_QUADS_LZS_TEST",
 281         "FRAG_QUADS_LZS_KILLED",
 282         "FRAG_CYCLE_NO_TILE",
 283         "FRAG_NUM_TILES",
 284         "FRAG_TRANS_ELIM",
 285         "COMPUTE_ACTIVE",
 286         "COMPUTE_TASKS",
 287         "COMPUTE_THREADS",
 288         "COMPUTE_CYCLES_DESC",
 289         "TRIPIPE_ACTIVE",
 290         "ARITH_WORDS",
 291         "ARITH_CYCLES_REG",
 292         "ARITH_CYCLES_L0",
 293         "ARITH_FRAG_DEPEND",
 294         "LS_WORDS",
 295         "LS_ISSUES",
 296         "LS_RESTARTS",
 297         "LS_REISSUES_MISS",
 298         "LS_REISSUES_VD",
 299         "LS_REISSUE_ATTRIB_MISS",
 300         "LS_NO_WB",
 301         "TEX_WORDS",
 302         "TEX_BUBBLES",
 303         "TEX_WORDS_L0",
 304         "TEX_WORDS_DESC",
 305         "TEX_THREADS",
 306         "TEX_RECIRC_FMISS",
 307         "TEX_RECIRC_DESC",
 308         "TEX_RECIRC_MULTI",
 309         "TEX_RECIRC_PMISS",
 310         "TEX_RECIRC_CONF",
 311         "LSC_READ_HITS",
 312         "LSC_READ_MISSES",
 313         "LSC_WRITE_HITS",
 314         "LSC_WRITE_MISSES",
 315         "LSC_ATOMIC_HITS",
 316         "LSC_ATOMIC_MISSES",
 317         "LSC_LINE_FETCHES",
 318         "LSC_DIRTY_LINE",
 319         "LSC_SNOOPS",
 320         "AXI_TLB_STALL",
 321         "AXI_TLB_MIESS",
 322         "AXI_TLB_TRANSACTION",
 323         "LS_TLB_MISS",
 324         "LS_TLB_HIT",
 325         "AXI_BEATS_READ",
 326         "AXI_BEATS_WRITTEN",
 327
 328         /*L2 and MMU */
 329         "",
 330         "",
 331         "",
 332         "",
 333         "MMU_HIT",
 334         "MMU_NEW_MISS",
 335         "MMU_REPLAY_FULL",
 336         "MMU_REPLAY_MISS",
 337         "MMU_TABLE_WALK",
 338         "",
 339         "",
 340         "",
 341         "",
 342         "",
 343         "",
 344         "",
 345         "UTLB_HIT",
 346         "UTLB_NEW_MISS",
 347         "UTLB_REPLAY_FULL",
 348         "UTLB_REPLAY_MISS",
 349         "UTLB_STALL",
 350         "",
 351         "",
 352         "",
 353         "",
 354         "",
 355         "",
 356         "",
 357         "",
 358         "",
 359         "L2_WRITE_BEATS",
 360         "L2_READ_BEATS",
 361         "L2_ANY_LOOKUP",
 362         "L2_READ_LOOKUP",
 363         "L2_SREAD_LOOKUP",
 364         "L2_READ_REPLAY",
 365         "L2_READ_SNOOP",
 366         "L2_READ_HIT",
 367         "L2_CLEAN_MISS",
 368         "L2_WRITE_LOOKUP",
 369         "L2_SWRITE_LOOKUP",
 370         "L2_WRITE_REPLAY",
 371         "L2_WRITE_SNOOP",
 372         "L2_WRITE_HIT",
 373         "L2_EXT_READ_FULL",
 374         "L2_EXT_READ_HALF",
 375         "L2_EXT_WRITE_FULL",
 376         "L2_EXT_WRITE_HALF",
 377         "L2_EXT_READ",
 378         "L2_EXT_READ_LINE",
 379         "L2_EXT_WRITE",
 380         "L2_EXT_WRITE_LINE",
 381         "L2_EXT_WRITE_SMALL",
 382         "L2_EXT_BARRIER",
 383         "L2_EXT_AR_STALL",
 384         "L2_EXT_R_BUF_FULL",
 385         "L2_EXT_RD_BUF_FULL",
 386         "L2_EXT_R_RAW",
 387         "L2_EXT_W_STALL",
 388         "L2_EXT_W_BUF_FULL",
 389         "L2_EXT_R_W_HAZARD",
 390         "L2_TAG_HAZARD",
 391         "L2_SNOOP_FULL",
 392         "L2_REPLAY_FULL"
 393 };
 394
 395 static const int number_of_hardware_counters = ARRAY_SIZE(hardware_counter_names);
 396
 397 #endif
 398
 399 #define GET_HW_BLOCK(c) (((c) >> 6) & 0x3)
 400 #define GET_COUNTER_OFFSET(c) ((c) & 0x3f)
 401
 402 #if MALI_DDK_GATOR_API_VERSION == 3
 403 /* Opaque handles for kbase_context and kbase_hwc_dma_mapping */
 404 static struct kbase_gator_hwcnt_handles *handles;
 405
 406 /* Information about hardware counters */
 407 static struct kbase_gator_hwcnt_info *in_out_info;
 408
 409 #else
 410 /* Memory to dump hardware counters into */
 411 static void *kernel_dump_buffer;
 412
 413 #if MALI_DDK_GATOR_API_VERSION == 2
 414 /* DMA state used to manage lifetime of the buffer */
 415 struct kbase_hwc_dma_mapping kernel_dump_buffer_handle;
 416 #endif
 417
 418 /* kbase context and device */
 419 static struct kbase_context *kbcontext;
 420 static struct kbase_device *kbdevice;
 421
 422 /*
 423  * The following function has no external prototype in older DDK
 424  * revisions. When the DDK is updated then this should be removed.
 425  */
 426 struct kbase_device *kbase_find_device(int minor);
 427 #endif
 428
 429 static volatile bool kbase_device_busy;
 430 static unsigned int num_hardware_counters_enabled;
 431
 432 /* gatorfs variables for counter enable state */
 433 static struct mali_counter *counters;
 434
 435 /* An array used to return the data we recorded as key,value pairs */
 436 static long long *counter_dump;
 437 static uint64_t last_read_time;
 438
 439 extern struct mali_counter mali_activity[3];
 440
 441 static const char *const mali_activity_names[] = {
 442         "fragment",
 443         "vertex",
 444         "opencl",
 445 };
 446
 447 #define SYMBOL_GET(FUNCTION, ERROR_COUNT) \
 448         do { \
 449                 if (FUNCTION ## _symbol) { \
 450                         pr_err("gator: mali " #FUNCTION " symbol was already registered\n"); \
 451                         (ERROR_COUNT)++; \
 452                 } else { \
 453                         FUNCTION ## _symbol = symbol_get(FUNCTION); \
 454                         if (!FUNCTION ## _symbol) { \
 455                                 pr_err("gator: mali online " #FUNCTION " symbol not found\n"); \
 456                                 (ERROR_COUNT)++; \
 457                         } \
 458                 } \
 459         } while (0)
 460
 461 #define SYMBOL_CLEANUP(FUNCTION) \
 462         do { \
 463                 if (FUNCTION ## _symbol) { \
 464                         symbol_put(FUNCTION); \
 465                         FUNCTION ## _symbol = NULL; \
 466                 } \
 467         } while (0)
 468
 469 /**
 470  * Execute symbol_get for all the Mali symbols and check for success.
 471  * @return the number of symbols not loaded.
 472  */
 473 static int init_symbols(void)
 474 {
 475         int error_count = 0;
 476 #if MALI_DDK_GATOR_API_VERSION == 3
 477         SYMBOL_GET(kbase_gator_instr_hwcnt_dump_irq, error_count);
 478         SYMBOL_GET(kbase_gator_instr_hwcnt_dump_complete, error_count);
 479         SYMBOL_GET(kbase_gator_hwcnt_init, error_count);
 480         SYMBOL_GET(kbase_gator_hwcnt_term, error_count);
 481 #else
 482         SYMBOL_GET(kbase_find_device, error_count);
 483         SYMBOL_GET(kbase_create_context, error_count);
 484         SYMBOL_GET(kbase_va_alloc, error_count);
 485         SYMBOL_GET(kbase_instr_hwcnt_enable, error_count);
 486         SYMBOL_GET(kbase_instr_hwcnt_clear, error_count);
 487         SYMBOL_GET(kbase_instr_hwcnt_dump_irq, error_count);
 488         SYMBOL_GET(kbase_instr_hwcnt_dump_complete, error_count);
 489         SYMBOL_GET(kbase_instr_hwcnt_disable, error_count);
 490         SYMBOL_GET(kbase_va_free, error_count);
 491         SYMBOL_GET(kbase_destroy_context, error_count);
 492 #endif
 493
 494         return error_count;
 495 }
 496
 497 /**
 498  * Execute symbol_put for all the registered Mali symbols.
 499  */
 500 static void clean_symbols(void)
 501 {
 502 #if MALI_DDK_GATOR_API_VERSION == 3
 503         SYMBOL_CLEANUP(kbase_gator_instr_hwcnt_dump_irq);
 504         SYMBOL_CLEANUP(kbase_gator_instr_hwcnt_dump_complete);
 505         SYMBOL_CLEANUP(kbase_gator_hwcnt_init);
 506         SYMBOL_CLEANUP(kbase_gator_hwcnt_term);
 507 #else
 508         SYMBOL_CLEANUP(kbase_find_device);
 509         SYMBOL_CLEANUP(kbase_create_context);
 510         SYMBOL_CLEANUP(kbase_va_alloc);
 511         SYMBOL_CLEANUP(kbase_instr_hwcnt_enable);
 512         SYMBOL_CLEANUP(kbase_instr_hwcnt_clear);
 513         SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_irq);
 514         SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_complete);
 515         SYMBOL_CLEANUP(kbase_instr_hwcnt_disable);
 516         SYMBOL_CLEANUP(kbase_va_free);
 517         SYMBOL_CLEANUP(kbase_destroy_context);
 518 #endif
 519 }
 520
 521 static int start(void)
 522 {
 523 #if MALI_DDK_GATOR_API_VERSION != 3
 524         struct kbase_uk_hwcnt_setup setup;
 525         unsigned long long shadersPresent = 0;
 526         u16 bitmask[] = { 0, 0, 0, 0 };
 527         mali_error err;
 528 #endif
 529         int cnt;
 530
 531         last_read_time = 0;
 532
 533 #if MALI_DDK_GATOR_API_VERSION == 3
 534         /* Setup HW counters */
 535         num_hardware_counters_enabled = 0;
 536
 537         /* Declare and initialise kbase_gator_hwcnt_info structure */
 538         in_out_info = kmalloc(sizeof(*in_out_info), GFP_KERNEL);
 539         for (cnt = 0; cnt < ARRAY_SIZE(in_out_info->bitmask); cnt++)
 540                 in_out_info->bitmask[cnt] = 0;
 541
 542         /* Calculate enable bitmasks based on counters_enabled array */
 543         for (cnt = 0; cnt < number_of_hardware_counters; cnt++) {
 544                 if (counters[cnt].enabled) {
 545                         int block = GET_HW_BLOCK(cnt);
 546                         int enable_bit = GET_COUNTER_OFFSET(cnt) / 4;
 547
 548                         in_out_info->bitmask[block] |= (1 << enable_bit);
 549                         pr_debug("gator: Mali-Midgard: hardware counter %s selected [%d]\n", hardware_counter_names[cnt], cnt);
 550                         num_hardware_counters_enabled++;
 551                 }
 552         }
 553
 554         /* Create a kbase context for HW counters */
 555         if (num_hardware_counters_enabled > 0) {
 556                 if (init_symbols() > 0) {
 557                         clean_symbols();
 558                         /* No Mali driver code entrypoints found - not a fault. */
 559                         return 0;
 560                 }
 561
 562                 handles = kbase_gator_hwcnt_init_symbol(in_out_info);
 563
 564                 if (handles == NULL)
 565                         goto out;
 566
 567                 kbase_device_busy = false;
 568         }
 569
 570         return 0;
 571 #else
 572         /* Setup HW counters */
 573         num_hardware_counters_enabled = 0;
 574
 575         /* Calculate enable bitmasks based on counters_enabled array */
 576         for (cnt = 0; cnt < number_of_hardware_counters; cnt++) {
 577                 const struct mali_counter *counter = &counters[cnt];
 578
 579                 if (counter->enabled) {
 580                         int block = GET_HW_BLOCK(cnt);
 581                         int enable_bit = GET_COUNTER_OFFSET(cnt) / 4;
 582
 583                         bitmask[block] |= (1 << enable_bit);
 584                         pr_debug("gator: Mali-Midgard: hardware counter %s selected [%d]\n", hardware_counter_names[cnt], cnt);
 585                         num_hardware_counters_enabled++;
 586                 }
 587         }
 588
 589         /* Create a kbase context for HW counters */
 590         if (num_hardware_counters_enabled > 0) {
 591                 if (init_symbols() > 0) {
 592                         clean_symbols();
 593                         /* No Mali driver code entrypoints found - not a fault. */
 594                         return 0;
 595                 }
 596
 597                 kbdevice = kbase_find_device_symbol(-1);
 598
 599                 /* If we already got a context, fail */
 600                 if (kbcontext) {
 601                         pr_err("gator: Mali-Midgard: error context already present\n");
 602                         goto out;
 603                 }
 604
 605                 /* kbcontext will only be valid after all the Mali symbols are loaded successfully */
 606                 kbcontext = kbase_create_context_symbol(kbdevice);
 607                 if (!kbcontext) {
 608                         pr_err("gator: Mali-Midgard: error creating kbase context\n");
 609                         goto out;
 610                 }
 611
 612                 /* See if we can get the number of shader cores */
 613                 shadersPresent = kbdevice->shader_present_bitmap;
 614                 shader_present_low = (unsigned long)shadersPresent;
 615
 616                 /*
 617                  * The amount of memory needed to store the dump (bytes)
 618                  * DUMP_SIZE = number of core groups
 619                  *             * number of blocks (always 8 for midgard)
 620                  *             * number of counters per block (always 64 for midgard)
 621                  *             * number of bytes per counter (always 4 in midgard)
 622                  * For a Mali-Midgard with a single core group = 1 * 8 * 64 * 4 = 2048
 623                  * For a Mali-Midgard with a dual core group   = 2 * 8 * 64 * 4 = 4096
 624                  */
 625 #if MALI_DDK_GATOR_API_VERSION == 1
 626                 kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096);
 627 #elif MALI_DDK_GATOR_API_VERSION == 2
 628                 kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096, &kernel_dump_buffer_handle);
 629 #endif
 630                 if (!kernel_dump_buffer) {
 631                         pr_err("gator: Mali-Midgard: error trying to allocate va\n");
 632                         goto destroy_context;
 633                 }
 634
 635                 setup.dump_buffer = (uintptr_t)kernel_dump_buffer;
 636                 setup.jm_bm = bitmask[JM_BLOCK];
 637                 setup.tiler_bm = bitmask[TILER_BLOCK];
 638                 setup.shader_bm = bitmask[SHADER_BLOCK];
 639                 setup.mmu_l2_bm = bitmask[MMU_BLOCK];
 640                 /* These counters do not exist on Mali-T60x */
 641                 setup.l3_cache_bm = 0;
 642
 643                 /* Use kbase API to enable hardware counters and provide dump buffer */
 644                 err = kbase_instr_hwcnt_enable_symbol(kbcontext, &setup);
 645                 if (err != MALI_ERROR_NONE) {
 646                         pr_err("gator: Mali-Midgard: can't setup hardware counters\n");
 647                         goto free_buffer;
 648                 }
 649                 pr_debug("gator: Mali-Midgard: hardware counters enabled\n");
 650                 kbase_instr_hwcnt_clear_symbol(kbcontext);
 651                 pr_debug("gator: Mali-Midgard: hardware counters cleared\n");
 652
 653                 kbase_device_busy = false;
 654         }
 655
 656         return 0;
 657
 658 free_buffer:
 659 #if MALI_DDK_GATOR_API_VERSION == 1
 660         kbase_va_free_symbol(kbcontext, kernel_dump_buffer);
 661 #elif MALI_DDK_GATOR_API_VERSION == 2
 662         kbase_va_free_symbol(kbcontext, &kernel_dump_buffer_handle);
 663 #endif
 664
 665 destroy_context:
 666         kbase_destroy_context_symbol(kbcontext);
 667 #endif
 668
 669 out:
 670         clean_symbols();
 671         return -1;
 672 }
 673
 674 static void stop(void)
 675 {
 676         unsigned int cnt;
 677 #if MALI_DDK_GATOR_API_VERSION == 3
 678         struct kbase_gator_hwcnt_handles *temp_hand;
 679 #else
 680         struct kbase_context *temp_kbcontext;
 681 #endif
 682
 683         pr_debug("gator: Mali-Midgard: stop\n");
 684
 685         /* Set all counters as disabled */
 686         for (cnt = 0; cnt < number_of_hardware_counters; cnt++)
 687                 counters[cnt].enabled = 0;
 688
 689         /* Destroy the context for HW counters */
 690 #if MALI_DDK_GATOR_API_VERSION == 3
 691         if (num_hardware_counters_enabled > 0 && handles != NULL) {
 692                 /*
 693                  * Set the global variable to NULL before destroying it, because
 694                  * other function will check this before using it.
 695                  */
 696                 temp_hand = handles;
 697                 handles = NULL;
 698
 699                 kbase_gator_hwcnt_term_symbol(in_out_info, temp_hand);
 700
 701                 kfree(in_out_info);
 702
 703 #else
 704         if (num_hardware_counters_enabled > 0 && kbcontext != NULL) {
 705                 /*
 706                  * Set the global variable to NULL before destroying it, because
 707                  * other function will check this before using it.
 708                  */
 709                 temp_kbcontext = kbcontext;
 710                 kbcontext = NULL;
 711
 712                 kbase_instr_hwcnt_disable_symbol(temp_kbcontext);
 713
 714 #if MALI_DDK_GATOR_API_VERSION == 1
 715                 kbase_va_free_symbol(temp_kbcontext, kernel_dump_buffer);
 716 #elif MALI_DDK_GATOR_API_VERSION == 2
 717                 kbase_va_free_symbol(temp_kbcontext, &kernel_dump_buffer_handle);
 718 #endif
 719
 720                 kbase_destroy_context_symbol(temp_kbcontext);
 721 #endif
 722
 723                 pr_debug("gator: Mali-Midgard: hardware counters stopped\n");
 724
 725                 clean_symbols();
 726         }
 727 }
 728
 729 static int read_counter(const int cnt, const int len, const struct mali_counter *counter)
 730 {
 731         const int block = GET_HW_BLOCK(cnt);
 732         const int counter_offset = GET_COUNTER_OFFSET(cnt);
 733         u32 value = 0;
 734
 735 #if MALI_DDK_GATOR_API_VERSION == 3
 736         const char *block_base_address = (char *)in_out_info->kernel_dump_buffer;
 737         int i;
 738         int shader_core_count = 0;
 739
 740         for (i = 0; i < in_out_info->nr_hwc_blocks; i++) {
 741                 if (block == in_out_info->hwc_layout[i]) {
 742                         value += *((u32 *)(block_base_address + (0x100 * i)) + counter_offset);
 743                         if (block == SHADER_BLOCK)
 744                                 ++shader_core_count;
 745                 }
 746         }
 747
 748         if (shader_core_count > 1)
 749                 value /= shader_core_count;
 750 #else
 751         const unsigned int vithar_blocks[] = {
 752                 0x700,  /* VITHAR_JOB_MANAGER,     Block 0 */
 753                 0x400,  /* VITHAR_TILER,           Block 1 */
 754                 0x000,  /* VITHAR_SHADER_CORE,     Block 2 */
 755                 0x500   /* VITHAR_MEMORY_SYSTEM,   Block 3 */
 756         };
 757         const char *block_base_address = (char *)kernel_dump_buffer + vithar_blocks[block];
 758
 759         /* If counter belongs to shader block need to take into account all cores */
 760         if (block == SHADER_BLOCK) {
 761                 int i = 0;
 762                 int shader_core_count = 0;
 763
 764                 value = 0;
 765
 766                 for (i = 0; i < 4; i++) {
 767                         if ((shader_present_low >> i) & 1) {
 768                                 value += *((u32 *)(block_base_address + (0x100 * i)) + counter_offset);
 769                                 shader_core_count++;
 770                         }
 771                 }
 772
 773                 for (i = 0; i < 4; i++) {
 774                         if ((shader_present_low >> (i+4)) & 1) {
 775                                 value += *((u32 *)(block_base_address + (0x100 * i) + 0x800) + counter_offset);
 776                                 shader_core_count++;
 777                         }
 778                 }
 779
 780                 /* Need to total by number of cores to produce an average */
 781                 if (shader_core_count != 0)
 782                         value /= shader_core_count;
 783         } else {
 784                 value = *((u32 *)block_base_address + counter_offset);
 785         }
 786 #endif
 787
 788         counter_dump[len + 0] = counter->key;
 789         counter_dump[len + 1] = value;
 790
 791         return 2;
 792 }
 793
 794 static int read(long long **buffer, bool sched_switch)
 795 {
 796         int cnt;
 797         int len = 0;
 798         uint32_t success;
 799
 800         uint64_t curr_time;
 801
 802         if (!on_primary_core() || sched_switch)
 803                 return 0;
 804
 805         /*
 806          * Report the HW counters
 807          * Only process hardware counters if at least one of the hardware counters is enabled.
 808          */
 809         if (num_hardware_counters_enabled <= 0)
 810                 return 0;
 811
 812         curr_time = gator_get_time();
 813
 814         /*
 815          * Discard reads unless a respectable time has passed. This
 816          * reduces the load on the GPU without sacrificing accuracy on
 817          * the Streamline display.
 818          */
 819         if (curr_time - last_read_time < READ_INTERVAL_NSEC)
 820                 return 0;
 821
 822 #if MALI_DDK_GATOR_API_VERSION == 3
 823         if (!handles)
 824                 return -1;
 825
 826         /* Mali symbols can be called safely since a kbcontext is valid */
 827         if (kbase_gator_instr_hwcnt_dump_complete_symbol(handles, &success)) {
 828 #else
 829         if (!kbcontext)
 830                 return -1;
 831
 832         /* Mali symbols can be called safely since a kbcontext is valid */
 833         if (kbase_instr_hwcnt_dump_complete_symbol(kbcontext, &success)) {
 834 #endif
 835                 kbase_device_busy = false;
 836
 837                 /*
 838                  * If last_read_time is zero, then this result is from a previous
 839                  * capture or in error.
 840                  */
 841                 if (success && last_read_time > 0) {
 842                         /* Backdate these events to when they were requested */
 843                         counter_dump[len++] = 0;
 844                         counter_dump[len++] = last_read_time;
 845
 846                         /* Cycle through hardware counters and accumulate totals */
 847                         for (cnt = 0; cnt < number_of_hardware_counters; cnt++) {
 848                                 const struct mali_counter *counter = &counters[cnt];
 849
 850                                 if (counter->enabled)
 851                                         len += read_counter(cnt, len, counter);
 852                         }
 853
 854                         /* Restore the timestamp */
 855                         counter_dump[len++] = 0;
 856                         counter_dump[len++] = curr_time;
 857                 }
 858         }
 859
 860         if (!kbase_device_busy) {
 861                 kbase_device_busy = true;
 862                 last_read_time = curr_time;
 863 #if MALI_DDK_GATOR_API_VERSION == 3
 864                 kbase_gator_instr_hwcnt_dump_irq_symbol(handles);
 865 #else
 866                 kbase_instr_hwcnt_dump_irq_symbol(kbcontext);
 867 #endif
 868         }
 869
 870         /* Update the buffer */
 871         if (buffer)
 872                 *buffer = counter_dump;
 873
 874         return len;
 875 }
 876
 877 static int create_files(struct super_block *sb, struct dentry *root)
 878 {
 879         unsigned int event;
 880         /*
 881          * Create the filesystem for all events
 882          */
 883         for (event = 0; event < ARRAY_SIZE(mali_activity); event++) {
 884                 if (gator_mali_create_file_system("Midgard", mali_activity_names[event], sb, root, &mali_activity[event], NULL) != 0)
 885                         return -1;
 886         }
 887
 888         for (event = 0; event < number_of_hardware_counters; event++) {
 889                 if (gator_mali_create_file_system(mali_name, hardware_counter_names[event], sb, root, &counters[event], NULL) != 0)
 890                         return -1;
 891         }
 892
 893         return 0;
 894 }
 895
 896 static void shutdown(void)
 897 {
 898 #if MALI_DDK_GATOR_API_VERSION == 3
 899         void (*kbase_gator_hwcnt_term_names_symbol)(void) = NULL;
 900         int error_count = 0;
 901 #endif
 902
 903         kfree(counters);
 904         kfree(counter_dump);
 905
 906 #if MALI_DDK_GATOR_API_VERSION == 3
 907         SYMBOL_GET(kbase_gator_hwcnt_term_names, error_count);
 908
 909         number_of_hardware_counters = -1;
 910         hardware_counter_names = NULL;
 911         if (kbase_gator_hwcnt_term_names_symbol != NULL) {
 912                 kbase_gator_hwcnt_term_names_symbol();
 913                 pr_debug("gator: Released symbols\n");
 914         }
 915
 916         SYMBOL_CLEANUP(kbase_gator_hwcnt_term_names);
 917 #endif
 918 }
 919
 920 static struct gator_interface gator_events_mali_midgard_interface = {
 921         .name = "mali_midgard_hw",
 922         .shutdown = shutdown,
 923         .create_files = create_files,
 924         .start = start,
 925         .stop = stop,
 926         .read64 = read
 927 };
 928
 929 int gator_events_mali_midgard_hw_init(void)
 930 {
 931 #if MALI_DDK_GATOR_API_VERSION == 3
 932         const char *const *(*kbase_gator_hwcnt_init_names_symbol)(uint32_t *) = NULL;
 933         int error_count = 0;
 934 #endif
 935
 936         pr_debug("gator: Mali-Midgard: sw_counters init\n");
 937
 938 #if GATOR_TEST
 939         test_all_is_read_scheduled();
 940 #endif
 941
 942 #if MALI_DDK_GATOR_API_VERSION == 3
 943         SYMBOL_GET(kbase_gator_hwcnt_init_names, error_count);
 944         if (error_count > 0) {
 945                 SYMBOL_CLEANUP(kbase_gator_hwcnt_init_names);
 946                 return 1;
 947         }
 948
 949         number_of_hardware_counters = -1;
 950         hardware_counter_names = kbase_gator_hwcnt_init_names_symbol(&number_of_hardware_counters);
 951
 952         SYMBOL_CLEANUP(kbase_gator_hwcnt_init_names);
 953
 954         if ((hardware_counter_names == NULL) || (number_of_hardware_counters <= 0)) {
 955                 pr_err("gator: Error reading hardware counters names: got %d names\n", number_of_hardware_counters);
 956                 return -1;
 957         }
 958 #else
 959         mali_name = "Midgard";
 960 #endif
 961
 962         counters = kmalloc(sizeof(*counters)*number_of_hardware_counters, GFP_KERNEL);
 963         counter_dump = kmalloc(sizeof(*counter_dump)*number_of_hardware_counters*2 + 4, GFP_KERNEL);
 964
 965         gator_mali_initialise_counters(mali_activity, ARRAY_SIZE(mali_activity));
 966         gator_mali_initialise_counters(counters, number_of_hardware_counters);
 967
 968         return gator_events_install(&gator_events_mali_midgard_interface);
 969 }