drivers/gator/gator_main.c

   1 /**
   2  * Copyright (C) ARM Limited 2010-2015. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 /* This version must match the gator daemon version */
  11 #define PROTOCOL_VERSION 231
  12 static unsigned long gator_protocol_version = PROTOCOL_VERSION;
  13
  14 #include <linux/slab.h>
  15 #include <linux/cpu.h>
  16 #include <linux/sched.h>
  17 #include <linux/irq.h>
  18 #include <linux/vmalloc.h>
  19 #include <linux/hardirq.h>
  20 #include <linux/highmem.h>
  21 #include <linux/pagemap.h>
  22 #include <linux/suspend.h>
  23 #include <linux/module.h>
  24 #include <linux/perf_event.h>
  25 #include <linux/utsname.h>
  26 #include <linux/kthread.h>
  27 #include <linux/uaccess.h>
  28
  29 #include "gator.h"
  30 #include "gator_src_md5.h"
  31
  32 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
  33 #error Kernels prior to 3.4 not supported. DS-5 v5.21 and earlier supported 2.6.32 and later.
  34 #endif
  35
  36 #if defined(MODULE) && !defined(CONFIG_MODULES)
  37 #error Cannot build a module against a kernel that does not support modules. To resolve, either rebuild the kernel to support modules or build gator as part of the kernel.
  38 #endif
  39
  40 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
  41 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
  42 #endif
  43
  44 #ifndef CONFIG_PROFILING
  45 #error gator requires the kernel to have CONFIG_PROFILING defined
  46 #endif
  47
  48 #ifndef CONFIG_HIGH_RES_TIMERS
  49 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
  50 #endif
  51
  52 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) && defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
  53 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
  54 #endif
  55
  56 #if !(GATOR_PERF_PMU_SUPPORT)
  57 #ifndef CONFIG_PERF_EVENTS
  58 #error gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
  59 #elif !defined CONFIG_HW_PERF_EVENTS
  60 #error gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
  61 #endif
  62 #endif
  63
  64 /******************************************************************************
  65  * DEFINES
  66  ******************************************************************************/
  67 #define SUMMARY_BUFFER_SIZE       (1*1024)
  68 #define BACKTRACE_BUFFER_SIZE     (128*1024)
  69 #define NAME_BUFFER_SIZE          (64*1024)
  70 #define COUNTER_BUFFER_SIZE       (64*1024)     /* counters have the core as part of the data and the core value in the frame header may be discarded */
  71 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
  72 #define ANNOTATE_BUFFER_SIZE      (128*1024)    /* annotate counters have the core as part of the data and the core value in the frame header may be discarded */
  73 #define SCHED_TRACE_BUFFER_SIZE   (128*1024)
  74 #define IDLE_BUFFER_SIZE          (32*1024)     /* idle counters have the core as part of the data and the core value in the frame header may be discarded */
  75 #define ACTIVITY_BUFFER_SIZE      (128*1024)
  76
  77 #define NO_COOKIE      0U
  78 #define UNRESOLVED_COOKIE ~0U
  79
  80 #define FRAME_SUMMARY       1
  81 #define FRAME_BACKTRACE     2
  82 #define FRAME_NAME          3
  83 #define FRAME_COUNTER       4
  84 #define FRAME_BLOCK_COUNTER 5
  85 #define FRAME_ANNOTATE      6
  86 #define FRAME_SCHED_TRACE   7
  87 #define FRAME_IDLE          9
  88 #define FRAME_ACTIVITY     13
  89
  90 #define MESSAGE_END_BACKTRACE 1
  91
  92 /* Name Frame Messages */
  93 #define MESSAGE_COOKIE      1
  94 #define MESSAGE_THREAD_NAME 2
  95
  96 /* Scheduler Trace Frame Messages */
  97 #define MESSAGE_SCHED_SWITCH 1
  98 #define MESSAGE_SCHED_EXIT   2
  99
 100 /* Summary Frame Messages */
 101 #define MESSAGE_SUMMARY   1
 102 #define MESSAGE_CORE_NAME 3
 103
 104 /* Activity Frame Messages */
 105 #define MESSAGE_LINK   1
 106 #define MESSAGE_SWITCH 2
 107 #define MESSAGE_EXIT   3
 108
 109 #define MAXSIZE_PACK32     5
 110 #define MAXSIZE_PACK64    10
 111
 112 #define FRAME_HEADER_SIZE 3
 113
 114 #if defined(__arm__)
 115 #define PC_REG regs->ARM_pc
 116 #elif defined(__aarch64__)
 117 #define PC_REG regs->pc
 118 #else
 119 #define PC_REG regs->ip
 120 #endif
 121
 122 enum {
 123         SUMMARY_BUF,
 124         BACKTRACE_BUF,
 125         NAME_BUF,
 126         COUNTER_BUF,
 127         BLOCK_COUNTER_BUF,
 128         ANNOTATE_BUF,
 129         SCHED_TRACE_BUF,
 130         IDLE_BUF,
 131         ACTIVITY_BUF,
 132         NUM_GATOR_BUFS
 133 };
 134
 135 /******************************************************************************
 136  * Globals
 137  ******************************************************************************/
 138 static unsigned long gator_cpu_cores;
 139 /* Size of the largest buffer. Effectively constant, set in gator_op_create_files */
 140 static unsigned long userspace_buffer_size;
 141 static unsigned long gator_backtrace_depth;
 142 /* How often to commit the buffers for live in nanoseconds */
 143 static u64 gator_live_rate;
 144
 145 static unsigned long gator_started;
 146 static u64 gator_monotonic_started;
 147 static u64 gator_sync_time;
 148 static u64 gator_hibernate_time;
 149 static unsigned long gator_buffer_opened;
 150 static unsigned long gator_timer_count;
 151 static unsigned long gator_response_type;
 152 static DEFINE_MUTEX(start_mutex);
 153 static DEFINE_MUTEX(gator_buffer_mutex);
 154
 155 static bool event_based_sampling;
 156
 157 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
 158 static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait);
 159 static struct timer_list gator_buffer_wake_up_timer;
 160 static bool gator_buffer_wake_run;
 161 /* Initialize semaphore unlocked to initialize memory values */
 162 static DEFINE_SEMAPHORE(gator_buffer_wake_sem);
 163 static struct task_struct *gator_buffer_wake_thread;
 164 static LIST_HEAD(gator_events);
 165
 166 static DEFINE_PER_CPU(u64, last_timestamp);
 167
 168 static bool printed_monotonic_warning;
 169
 170 static u32 gator_cpuids[NR_CPUS];
 171 static bool sent_core_name[NR_CPUS];
 172
 173 static DEFINE_PER_CPU(bool, in_scheduler_context);
 174
 175 /******************************************************************************
 176  * Prototypes
 177  ******************************************************************************/
 178 static void gator_emit_perf_time(u64 time);
 179 static void gator_op_create_files(struct super_block *sb, struct dentry *root);
 180 static void gator_backtrace_handler(struct pt_regs *const regs);
 181 static int gator_events_perf_pmu_reread(void);
 182 static int gator_events_perf_pmu_create_files(struct super_block *sb, struct dentry *root);
 183
 184 /* gator_buffer is protected by being per_cpu and by having IRQs
 185  * disabled when writing to it. Most marshal_* calls take care of this
 186  * except for marshal_cookie*, marshal_backtrace* and marshal_frame
 187  * where the caller is responsible for doing so. No synchronization is
 188  * needed with the backtrace buffer as it is per cpu and is only used
 189  * from the hrtimer. The annotate_lock must be held when using the
 190  * annotation buffer as it is not per cpu. collect_counters which is
 191  * the sole writer to the block counter frame is additionally
 192  * protected by the per cpu collecting flag.
 193  */
 194
 195 /* Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup. */
 196 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
 197 /* gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup. */
 198 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
 199 /* Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read */
 200 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
 201 /* Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer */
 202 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
 203 /* Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace */
 204 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
 205 /* If set to false, decreases the number of bytes returned by
 206  * buffer_bytes_available. Set in buffer_check_space if no space is
 207  * remaining. Initialized to true in gator_op_setup. This means that
 208  * if we run out of space, continue to report that no space is
 209  * available until bytes are read by userspace
 210  */
 211 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
 212 /* The buffer. Allocated in gator_op_setup */
 213 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
 214 /* The time after which the buffer should be committed for live display */
 215 static DEFINE_PER_CPU(u64, gator_buffer_commit_time);
 216
 217 /* List of all gator events - new events must be added to this list */
 218 #define GATOR_EVENTS_LIST \
 219         GATOR_EVENT(gator_events_block_init) \
 220         GATOR_EVENT(gator_events_ccn504_init) \
 221         GATOR_EVENT(gator_events_irq_init) \
 222         GATOR_EVENT(gator_events_l2c310_init) \
 223         GATOR_EVENT(gator_events_mali_init) \
 224         GATOR_EVENT(gator_events_mali_midgard_hw_init) \
 225         GATOR_EVENT(gator_events_mali_midgard_init) \
 226         GATOR_EVENT(gator_events_meminfo_init) \
 227         GATOR_EVENT(gator_events_mmapped_init) \
 228         GATOR_EVENT(gator_events_net_init) \
 229         GATOR_EVENT(gator_events_perf_pmu_init) \
 230         GATOR_EVENT(gator_events_sched_init) \
 231
 232 #define GATOR_EVENT(EVENT_INIT) __weak int EVENT_INIT(void);
 233 GATOR_EVENTS_LIST
 234 #undef GATOR_EVENT
 235
 236 static int (*gator_events_list[])(void) = {
 237 #define GATOR_EVENT(EVENT_INIT) EVENT_INIT,
 238 GATOR_EVENTS_LIST
 239 #undef GATOR_EVENT
 240 };
 241
 242 /******************************************************************************
 243  * Application Includes
 244  ******************************************************************************/
 245 #include "gator_fs.c"
 246 #include "gator_pmu.c"
 247 #include "gator_buffer_write.c"
 248 #include "gator_buffer.c"
 249 #include "gator_marshaling.c"
 250 #include "gator_hrtimer_gator.c"
 251 #include "gator_cookies.c"
 252 #include "gator_annotate.c"
 253 #include "gator_trace_sched.c"
 254 #include "gator_trace_power.c"
 255 #include "gator_trace_gpu.c"
 256 #include "gator_backtrace.c"
 257 #include "gator_events_perf_pmu.c"
 258
 259 /******************************************************************************
 260  * Misc
 261  ******************************************************************************/
 262
 263 MODULE_PARM_DESC(gator_src_md5, "Gator driver source code md5sum");
 264 module_param_named(src_md5, gator_src_md5, charp, 0444);
 265
 266 u32 gator_cpuid(void)
 267 {
 268 #if defined(__arm__) || defined(__aarch64__)
 269         u32 val;
 270 #if !defined(__aarch64__)
 271         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
 272 #else
 273         asm volatile("mrs %0, midr_el1" : "=r" (val));
 274 #endif
 275         return ((val & 0xff000000) >> 12) | ((val & 0xfff0) >> 4);
 276 #else
 277         return OTHER;
 278 #endif
 279 }
 280
 281 static void gator_buffer_wake_up(unsigned long data)
 282 {
 283         wake_up(&gator_buffer_wait);
 284 }
 285
 286 static int gator_buffer_wake_func(void *data)
 287 {
 288         for (;;) {
 289                 if (down_killable(&gator_buffer_wake_sem))
 290                         break;
 291
 292                 /* Eat up any pending events */
 293                 while (!down_trylock(&gator_buffer_wake_sem))
 294                         ;
 295
 296                 if (!gator_buffer_wake_run)
 297                         break;
 298
 299                 gator_buffer_wake_up(0);
 300         }
 301
 302         return 0;
 303 }
 304
 305 /******************************************************************************
 306  * Commit interface
 307  ******************************************************************************/
 308 static bool buffer_commit_ready(int *cpu, int *buftype)
 309 {
 310         int cpu_x, x;
 311
 312         for_each_present_cpu(cpu_x) {
 313                 for (x = 0; x < NUM_GATOR_BUFS; x++)
 314                         if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
 315                                 *cpu = cpu_x;
 316                                 *buftype = x;
 317                                 return true;
 318                         }
 319         }
 320         *cpu = -1;
 321         *buftype = -1;
 322         return false;
 323 }
 324
 325 /******************************************************************************
 326  * hrtimer interrupt processing
 327  ******************************************************************************/
 328 static void gator_timer_interrupt(void)
 329 {
 330         struct pt_regs *const regs = get_irq_regs();
 331
 332         gator_backtrace_handler(regs);
 333 }
 334
 335 static void gator_backtrace_handler(struct pt_regs *const regs)
 336 {
 337         u64 time = gator_get_time();
 338         int cpu = get_physical_cpu();
 339
 340         /* Output backtrace */
 341         gator_add_sample(cpu, regs, time);
 342
 343         /* Collect counters */
 344         if (!per_cpu(collecting, cpu))
 345                 collect_counters(time, current, false);
 346
 347         /* No buffer flushing occurs during sched switch for RT-Preempt full. The block counter frame will be flushed by collect_counters, but the sched buffer needs to be explicitly flushed */
 348 #ifdef CONFIG_PREEMPT_RT_FULL
 349         buffer_check(cpu, SCHED_TRACE_BUF, time);
 350 #endif
 351 }
 352
 353 static int gator_running;
 354
 355 /* This function runs in interrupt context and on the appropriate core */
 356 static void gator_timer_offline(void *migrate)
 357 {
 358         struct gator_interface *gi;
 359         int i, len, cpu = get_physical_cpu();
 360         int *buffer;
 361         u64 time;
 362
 363         gator_trace_sched_offline();
 364         gator_trace_power_offline();
 365
 366         if (!migrate)
 367                 gator_hrtimer_offline();
 368
 369         /* Offline any events and output counters */
 370         time = gator_get_time();
 371         if (marshal_event_header(time)) {
 372                 list_for_each_entry(gi, &gator_events, list) {
 373                         if (gi->offline) {
 374                                 len = gi->offline(&buffer, migrate);
 375                                 if (len < 0)
 376                                         pr_err("gator: offline failed for %s\n", gi->name);
 377                                 marshal_event(len, buffer);
 378                         }
 379                 }
 380                 /* Only check after writing all counters so that time and corresponding counters appear in the same frame */
 381                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
 382         }
 383
 384         /* Flush all buffers on this core */
 385         for (i = 0; i < NUM_GATOR_BUFS; i++)
 386                 gator_commit_buffer(cpu, i, time);
 387 }
 388
 389 /* This function runs in interrupt context and may be running on a core other than core 'cpu' */
 390 static void gator_timer_offline_dispatch(int cpu, bool migrate)
 391 {
 392         struct gator_interface *gi;
 393
 394         list_for_each_entry(gi, &gator_events, list) {
 395                 if (gi->offline_dispatch)
 396                         gi->offline_dispatch(cpu, migrate);
 397         }
 398 }
 399
 400 static void gator_timer_stop(void)
 401 {
 402         int cpu;
 403
 404         if (gator_running) {
 405                 on_each_cpu(gator_timer_offline, NULL, 1);
 406                 for_each_online_cpu(cpu) {
 407                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
 408                 }
 409
 410                 gator_running = 0;
 411                 gator_hrtimer_shutdown();
 412         }
 413 }
 414
 415 static void gator_send_core_name(const int cpu, const u32 cpuid)
 416 {
 417 #if defined(__arm__) || defined(__aarch64__)
 418         if (!sent_core_name[cpu] || (cpuid != gator_cpuids[cpu])) {
 419                 const struct gator_cpu *const gator_cpu = gator_find_cpu_by_cpuid(cpuid);
 420                 const char *core_name = NULL;
 421                 char core_name_buf[32];
 422
 423                 /* Save off this cpuid */
 424                 gator_cpuids[cpu] = cpuid;
 425                 if (gator_cpu != NULL) {
 426                         core_name = gator_cpu->core_name;
 427                 } else {
 428                         if (cpuid == -1)
 429                                 snprintf(core_name_buf, sizeof(core_name_buf), "Unknown");
 430                         else
 431                                 snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.5x)", cpuid);
 432                         core_name = core_name_buf;
 433                 }
 434
 435                 marshal_core_name(cpu, cpuid, core_name);
 436                 sent_core_name[cpu] = true;
 437         }
 438 #endif
 439 }
 440
 441 static void gator_read_cpuid(void *arg)
 442 {
 443         gator_cpuids[get_physical_cpu()] = gator_cpuid();
 444 }
 445
 446 /* This function runs in interrupt context and on the appropriate core */
 447 static void gator_timer_online(void *migrate)
 448 {
 449         struct gator_interface *gi;
 450         int len, cpu = get_physical_cpu();
 451         int *buffer;
 452         u64 time;
 453
 454         /* Send what is currently running on this core */
 455         marshal_sched_trace_switch(current->pid, 0);
 456
 457         gator_trace_power_online();
 458
 459         /* online any events and output counters */
 460         time = gator_get_time();
 461         if (marshal_event_header(time)) {
 462                 list_for_each_entry(gi, &gator_events, list) {
 463                         if (gi->online) {
 464                                 len = gi->online(&buffer, migrate);
 465                                 if (len < 0)
 466                                         pr_err("gator: online failed for %s\n", gi->name);
 467                                 marshal_event(len, buffer);
 468                         }
 469                 }
 470                 /* Only check after writing all counters so that time and corresponding counters appear in the same frame */
 471                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
 472         }
 473
 474         if (!migrate)
 475                 gator_hrtimer_online();
 476
 477         gator_send_core_name(cpu, gator_cpuid());
 478 }
 479
 480 /* This function runs in interrupt context and may be running on a core other than core 'cpu' */
 481 static void gator_timer_online_dispatch(int cpu, bool migrate)
 482 {
 483         struct gator_interface *gi;
 484
 485         list_for_each_entry(gi, &gator_events, list) {
 486                 if (gi->online_dispatch)
 487                         gi->online_dispatch(cpu, migrate);
 488         }
 489 }
 490
 491 #include "gator_iks.c"
 492
 493 static int gator_timer_start(unsigned long sample_rate)
 494 {
 495         int cpu;
 496
 497         if (gator_running) {
 498                 pr_notice("gator: already running\n");
 499                 return 0;
 500         }
 501
 502         gator_running = 1;
 503
 504         /* event based sampling trumps hr timer based sampling */
 505         if (event_based_sampling)
 506                 sample_rate = 0;
 507
 508         if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
 509                 return -1;
 510
 511         /* Send off the previously saved cpuids */
 512         for_each_present_cpu(cpu) {
 513                 preempt_disable();
 514                 gator_send_core_name(cpu, gator_cpuids[cpu]);
 515                 preempt_enable();
 516         }
 517
 518         gator_send_iks_core_names();
 519         for_each_online_cpu(cpu) {
 520                 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
 521         }
 522         on_each_cpu(gator_timer_online, NULL, 1);
 523
 524         return 0;
 525 }
 526
 527 u64 gator_get_time(void)
 528 {
 529         struct timespec ts;
 530         u64 timestamp;
 531         u64 prev_timestamp;
 532         u64 delta;
 533         int cpu = smp_processor_id();
 534
 535         /* Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace */
 536         getrawmonotonic(&ts);
 537         timestamp = timespec_to_ns(&ts);
 538
 539         /* getrawmonotonic is not monotonic on all systems. Detect and
 540          * attempt to correct these cases. up to 0.5ms delta has been seen
 541          * on some systems, which can skew Streamline data when viewing at
 542          * high resolution. This doesn't work well with interrupts, but that
 543          * it's OK - the real concern is to catch big jumps in time
 544          */
 545         prev_timestamp = per_cpu(last_timestamp, cpu);
 546         if (prev_timestamp <= timestamp) {
 547                 per_cpu(last_timestamp, cpu) = timestamp;
 548         } else {
 549                 delta = prev_timestamp - timestamp;
 550                 /* Log the error once */
 551                 if (!printed_monotonic_warning && delta > 500000) {
 552                         pr_err("%s: getrawmonotonic is not monotonic  cpu: %i  delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __func__, cpu, delta);
 553                         printed_monotonic_warning = true;
 554                 }
 555                 timestamp = prev_timestamp;
 556         }
 557
 558         return timestamp - gator_monotonic_started;
 559 }
 560
 561 static void gator_emit_perf_time(u64 time)
 562 {
 563 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0)
 564         if (time >= gator_sync_time) {
 565                 marshal_event_single64(0, -1, local_clock());
 566                 gator_sync_time += NSEC_PER_SEC;
 567                 if (gator_live_rate <= 0)
 568                         gator_commit_buffer(get_physical_cpu(), COUNTER_BUF, time);
 569         }
 570 #endif
 571 }
 572
 573 /******************************************************************************
 574  * cpu hotplug and pm notifiers
 575  ******************************************************************************/
 576 static int gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
 577 {
 578         int cpu = lcpu_to_pcpu((long)hcpu);
 579
 580         switch (action) {
 581         case CPU_DOWN_PREPARE:
 582         case CPU_DOWN_PREPARE_FROZEN:
 583                 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
 584                 gator_timer_offline_dispatch(cpu, false);
 585                 break;
 586         case CPU_ONLINE:
 587         case CPU_ONLINE_FROZEN:
 588                 gator_timer_online_dispatch(cpu, false);
 589                 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
 590                 break;
 591         }
 592
 593         return NOTIFY_OK;
 594 }
 595
 596 static struct notifier_block __refdata gator_hotcpu_notifier = {
 597         .notifier_call = gator_hotcpu_notify,
 598 };
 599
 600 /* n.b. calling "on_each_cpu" only runs on those that are online.
 601  * Registered linux events are not disabled, so their counters will
 602  * continue to collect
 603  */
 604 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
 605 {
 606         int cpu;
 607         struct timespec ts;
 608
 609         switch (event) {
 610         case PM_HIBERNATION_PREPARE:
 611         case PM_SUSPEND_PREPARE:
 612                 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
 613                 unregister_scheduler_tracepoints();
 614                 on_each_cpu(gator_timer_offline, NULL, 1);
 615                 for_each_online_cpu(cpu) {
 616                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
 617                 }
 618
 619                 /* Record the wallclock hibernate time */
 620                 getnstimeofday(&ts);
 621                 gator_hibernate_time = timespec_to_ns(&ts) - gator_get_time();
 622                 break;
 623         case PM_POST_HIBERNATION:
 624         case PM_POST_SUSPEND:
 625                 /* Adjust gator_monotonic_started for the time spent sleeping, as gator_get_time does not account for it */
 626                 if (gator_hibernate_time > 0) {
 627                         getnstimeofday(&ts);
 628                         gator_monotonic_started += gator_hibernate_time + gator_get_time() - timespec_to_ns(&ts);
 629                         gator_hibernate_time = 0;
 630                 }
 631
 632                 for_each_online_cpu(cpu) {
 633                         gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
 634                 }
 635                 on_each_cpu(gator_timer_online, NULL, 1);
 636                 register_scheduler_tracepoints();
 637                 register_hotcpu_notifier(&gator_hotcpu_notifier);
 638                 break;
 639         }
 640
 641         return NOTIFY_OK;
 642 }
 643
 644 static struct notifier_block gator_pm_notifier = {
 645         .notifier_call = gator_pm_notify,
 646 };
 647
 648 static int gator_notifier_start(void)
 649 {
 650         int retval;
 651
 652         retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
 653         if (retval == 0)
 654                 retval = register_pm_notifier(&gator_pm_notifier);
 655         return retval;
 656 }
 657
 658 static void gator_notifier_stop(void)
 659 {
 660         unregister_pm_notifier(&gator_pm_notifier);
 661         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
 662 }
 663
 664 /******************************************************************************
 665  * Main
 666  ******************************************************************************/
 667 static void gator_summary(void)
 668 {
 669         u64 timestamp, uptime;
 670         struct timespec ts;
 671         char uname_buf[100];
 672
 673         snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine);
 674
 675         getnstimeofday(&ts);
 676         timestamp = timespec_to_ns(&ts);
 677
 678         /* Similar to reading /proc/uptime from fs/proc/uptime.c, calculate uptime */
 679 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0)
 680         {
 681                 void (*m2b)(struct timespec *ts);
 682
 683                 do_posix_clock_monotonic_gettime(&ts);
 684                 /* monotonic_to_bootbased is not defined for some versions of Android */
 685                 m2b = symbol_get(monotonic_to_bootbased);
 686                 if (m2b)
 687                         m2b(&ts);
 688         }
 689 #else
 690         get_monotonic_boottime(&ts);
 691 #endif
 692         uptime = timespec_to_ns(&ts);
 693
 694         /* Disable preemption as gator_get_time calls smp_processor_id to verify time is monotonic */
 695         preempt_disable();
 696         /* Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started */
 697         gator_monotonic_started = 0;
 698         gator_monotonic_started = gator_get_time();
 699
 700         marshal_summary(timestamp, uptime, gator_monotonic_started, uname_buf);
 701         gator_sync_time = 0;
 702         gator_emit_perf_time(gator_monotonic_started);
 703         /* Always flush COUNTER_BUF so that the initial perf_time is received before it's used */
 704         gator_commit_buffer(get_physical_cpu(), COUNTER_BUF, 0);
 705         preempt_enable();
 706 }
 707
 708 int gator_events_install(struct gator_interface *interface)
 709 {
 710         list_add_tail(&interface->list, &gator_events);
 711
 712         return 0;
 713 }
 714
 715 int gator_events_get_key(void)
 716 {
 717         /* key 0 is reserved as a timestamp. key 1 is reserved as the marker
 718          * for thread specific counters. key 2 is reserved as the marker for
 719          * core. Odd keys are assigned by the driver, even keys by the
 720          * daemon.
 721          */
 722         static int key = 3;
 723         const int ret = key;
 724
 725         key += 2;
 726         return ret;
 727 }
 728
 729 static int gator_init(void)
 730 {
 731         int i;
 732
 733         calc_first_cluster_size();
 734
 735         /* events sources */
 736         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
 737                 if (gator_events_list[i])
 738                         gator_events_list[i]();
 739
 740         gator_trace_sched_init();
 741         gator_trace_power_init();
 742
 743         return 0;
 744 }
 745
 746 static void gator_exit(void)
 747 {
 748         struct gator_interface *gi;
 749
 750         list_for_each_entry(gi, &gator_events, list)
 751                 if (gi->shutdown)
 752                         gi->shutdown();
 753 }
 754
 755 static int gator_start(void)
 756 {
 757         unsigned long cpu, i;
 758         struct gator_interface *gi;
 759
 760         gator_buffer_wake_run = true;
 761         gator_buffer_wake_thread = kthread_run(gator_buffer_wake_func, NULL, "gator_bwake");
 762         if (IS_ERR(gator_buffer_wake_thread))
 763                 goto bwake_failure;
 764
 765         if (gator_migrate_start())
 766                 goto migrate_failure;
 767
 768         /* Initialize the buffer with the frame type and core */
 769         for_each_present_cpu(cpu) {
 770                 for (i = 0; i < NUM_GATOR_BUFS; i++)
 771                         marshal_frame(cpu, i);
 772                 per_cpu(last_timestamp, cpu) = 0;
 773         }
 774         printed_monotonic_warning = false;
 775
 776         /* Capture the start time */
 777         gator_summary();
 778
 779         /* start all events */
 780         list_for_each_entry(gi, &gator_events, list) {
 781                 if (gi->start && gi->start() != 0) {
 782                         struct list_head *ptr = gi->list.prev;
 783
 784                         while (ptr != &gator_events) {
 785                                 gi = list_entry(ptr, struct gator_interface, list);
 786
 787                                 if (gi->stop)
 788                                         gi->stop();
 789
 790                                 ptr = ptr->prev;
 791                         }
 792                         goto events_failure;
 793                 }
 794         }
 795
 796         /* cookies shall be initialized before trace_sched_start() and gator_timer_start() */
 797         if (cookies_initialize())
 798                 goto cookies_failure;
 799         if (gator_annotate_start())
 800                 goto annotate_failure;
 801         if (gator_trace_sched_start())
 802                 goto sched_failure;
 803         if (gator_trace_power_start())
 804                 goto power_failure;
 805         if (gator_trace_gpu_start())
 806                 goto gpu_failure;
 807         if (gator_timer_start(gator_timer_count))
 808                 goto timer_failure;
 809         if (gator_notifier_start())
 810                 goto notifier_failure;
 811
 812         return 0;
 813
 814 notifier_failure:
 815         gator_timer_stop();
 816 timer_failure:
 817         gator_trace_gpu_stop();
 818 gpu_failure:
 819         gator_trace_power_stop();
 820 power_failure:
 821         gator_trace_sched_stop();
 822 sched_failure:
 823         gator_annotate_stop();
 824 annotate_failure:
 825         cookies_release();
 826 cookies_failure:
 827         /* stop all events */
 828         list_for_each_entry(gi, &gator_events, list)
 829                 if (gi->stop)
 830                         gi->stop();
 831 events_failure:
 832         gator_migrate_stop();
 833 migrate_failure:
 834         gator_buffer_wake_run = false;
 835         up(&gator_buffer_wake_sem);
 836         gator_buffer_wake_thread = NULL;
 837 bwake_failure:
 838
 839         return -1;
 840 }
 841
 842 static void gator_stop(void)
 843 {
 844         struct gator_interface *gi;
 845
 846         gator_annotate_stop();
 847         gator_trace_sched_stop();
 848         gator_trace_power_stop();
 849         gator_trace_gpu_stop();
 850
 851         /* stop all interrupt callback reads before tearing down other interfaces */
 852         gator_notifier_stop();  /* should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined */
 853         gator_timer_stop();
 854
 855         /* stop all events */
 856         list_for_each_entry(gi, &gator_events, list)
 857                 if (gi->stop)
 858                         gi->stop();
 859
 860         gator_migrate_stop();
 861
 862         gator_buffer_wake_run = false;
 863         up(&gator_buffer_wake_sem);
 864         gator_buffer_wake_thread = NULL;
 865 }
 866
 867 /******************************************************************************
 868  * Filesystem
 869  ******************************************************************************/
 870 /* fopen("buffer") */
 871 static int gator_op_setup(void)
 872 {
 873         int err = 0;
 874         int cpu, i;
 875
 876         mutex_lock(&start_mutex);
 877
 878         gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
 879         gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
 880
 881         gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
 882         gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
 883
 884         gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
 885         gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
 886
 887         gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
 888         gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
 889
 890         gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
 891         gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
 892
 893         gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
 894         gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
 895
 896         gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
 897         gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
 898
 899         gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
 900         gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
 901
 902         gator_buffer_size[ACTIVITY_BUF] = ACTIVITY_BUFFER_SIZE;
 903         gator_buffer_mask[ACTIVITY_BUF] = ACTIVITY_BUFFER_SIZE - 1;
 904
 905         /* Initialize percpu per buffer variables */
 906         for (i = 0; i < NUM_GATOR_BUFS; i++) {
 907                 /* Verify buffers are a power of 2 */
 908                 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
 909                         err = -ENOEXEC;
 910                         goto setup_error;
 911                 }
 912
 913                 for_each_present_cpu(cpu) {
 914                         per_cpu(gator_buffer_read, cpu)[i] = 0;
 915                         per_cpu(gator_buffer_write, cpu)[i] = 0;
 916                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
 917                         per_cpu(buffer_space_available, cpu)[i] = true;
 918                         per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate;
 919
 920                         /* Annotation is a special case that only uses a single buffer */
 921                         if (cpu > 0 && i == ANNOTATE_BUF) {
 922                                 per_cpu(gator_buffer, cpu)[i] = NULL;
 923                                 continue;
 924                         }
 925
 926                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
 927                         if (!per_cpu(gator_buffer, cpu)[i]) {
 928                                 err = -ENOMEM;
 929                                 goto setup_error;
 930                         }
 931                 }
 932         }
 933
 934 setup_error:
 935         mutex_unlock(&start_mutex);
 936         return err;
 937 }
 938
 939 /* Actually start profiling (echo 1>/dev/gator/enable) */
 940 static int gator_op_start(void)
 941 {
 942         int err = 0;
 943
 944         mutex_lock(&start_mutex);
 945
 946         if (gator_started || gator_start())
 947                 err = -EINVAL;
 948         else
 949                 gator_started = 1;
 950
 951         mutex_unlock(&start_mutex);
 952
 953         return err;
 954 }
 955
 956 /* echo 0>/dev/gator/enable */
 957 static void gator_op_stop(void)
 958 {
 959         mutex_lock(&start_mutex);
 960
 961         if (gator_started) {
 962                 gator_stop();
 963
 964                 mutex_lock(&gator_buffer_mutex);
 965
 966                 gator_started = 0;
 967                 gator_monotonic_started = 0;
 968                 cookies_release();
 969                 wake_up(&gator_buffer_wait);
 970
 971                 mutex_unlock(&gator_buffer_mutex);
 972         }
 973
 974         mutex_unlock(&start_mutex);
 975 }
 976
 977 static void gator_shutdown(void)
 978 {
 979         int cpu, i;
 980
 981         mutex_lock(&start_mutex);
 982
 983         for_each_present_cpu(cpu) {
 984                 mutex_lock(&gator_buffer_mutex);
 985                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
 986                         vfree(per_cpu(gator_buffer, cpu)[i]);
 987                         per_cpu(gator_buffer, cpu)[i] = NULL;
 988                         per_cpu(gator_buffer_read, cpu)[i] = 0;
 989                         per_cpu(gator_buffer_write, cpu)[i] = 0;
 990                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
 991                         per_cpu(buffer_space_available, cpu)[i] = true;
 992                         per_cpu(gator_buffer_commit_time, cpu) = 0;
 993                 }
 994                 mutex_unlock(&gator_buffer_mutex);
 995         }
 996
 997         memset(&sent_core_name, 0, sizeof(sent_core_name));
 998
 999         mutex_unlock(&start_mutex);
1000 }
1001
1002 static int gator_set_backtrace(unsigned long val)
1003 {
1004         int err = 0;
1005
1006         mutex_lock(&start_mutex);
1007
1008         if (gator_started)
1009                 err = -EBUSY;
1010         else
1011                 gator_backtrace_depth = val;
1012
1013         mutex_unlock(&start_mutex);
1014
1015         return err;
1016 }
1017
1018 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1019 {
1020         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
1021 }
1022
1023 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1024 {
1025         unsigned long val;
1026         int retval;
1027
1028         if (*offset)
1029                 return -EINVAL;
1030
1031         retval = gatorfs_ulong_from_user(&val, buf, count);
1032         if (retval)
1033                 return retval;
1034
1035         if (val)
1036                 retval = gator_op_start();
1037         else
1038                 gator_op_stop();
1039
1040         if (retval)
1041                 return retval;
1042         return count;
1043 }
1044
1045 static const struct file_operations enable_fops = {
1046         .read = enable_read,
1047         .write = enable_write,
1048 };
1049
1050 static int userspace_buffer_open(struct inode *inode, struct file *file)
1051 {
1052         int err = -EPERM;
1053
1054         if (!capable(CAP_SYS_ADMIN))
1055                 return -EPERM;
1056
1057         if (test_and_set_bit_lock(0, &gator_buffer_opened))
1058                 return -EBUSY;
1059
1060         err = gator_op_setup();
1061         if (err)
1062                 goto fail;
1063
1064         /* NB: the actual start happens from userspace
1065          * echo 1 >/dev/gator/enable
1066          */
1067
1068         return 0;
1069
1070 fail:
1071         __clear_bit_unlock(0, &gator_buffer_opened);
1072         return err;
1073 }
1074
1075 static int userspace_buffer_release(struct inode *inode, struct file *file)
1076 {
1077         gator_op_stop();
1078         gator_shutdown();
1079         __clear_bit_unlock(0, &gator_buffer_opened);
1080         return 0;
1081 }
1082
1083 static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1084 {
1085         int commit, length1, length2, read;
1086         char *buffer1;
1087         char *buffer2;
1088         int cpu, buftype;
1089         int written = 0;
1090
1091         /* ensure there is enough space for a whole frame */
1092         if (count < userspace_buffer_size || *offset)
1093                 return -EINVAL;
1094
1095         /* sleep until the condition is true or a signal is received the
1096          * condition is checked each time gator_buffer_wait is woken up
1097          */
1098         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
1099
1100         if (signal_pending(current))
1101                 return -EINTR;
1102
1103         if (buftype == -1 || cpu == -1)
1104                 return 0;
1105
1106         mutex_lock(&gator_buffer_mutex);
1107
1108         do {
1109                 read = per_cpu(gator_buffer_read, cpu)[buftype];
1110                 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1111
1112                 /* May happen if the buffer is freed during pending reads. */
1113                 if (!per_cpu(gator_buffer, cpu)[buftype])
1114                         break;
1115
1116                 /* determine the size of two halves */
1117                 length1 = commit - read;
1118                 length2 = 0;
1119                 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1120                 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1121                 if (length1 < 0) {
1122                         length1 = gator_buffer_size[buftype] - read;
1123                         length2 = commit;
1124                 }
1125
1126                 if (length1 + length2 > count - written)
1127                         break;
1128
1129                 /* start, middle or end */
1130                 if (length1 > 0 && copy_to_user(&buf[written], buffer1, length1))
1131                         break;
1132
1133                 /* possible wrap around */
1134                 if (length2 > 0 && copy_to_user(&buf[written + length1], buffer2, length2))
1135                         break;
1136
1137                 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1138                 written += length1 + length2;
1139
1140                 /* Wake up annotate_write if more space is available */
1141                 if (buftype == ANNOTATE_BUF)
1142                         wake_up(&gator_annotate_wait);
1143         } while (buffer_commit_ready(&cpu, &buftype));
1144
1145         mutex_unlock(&gator_buffer_mutex);
1146
1147         /* kick just in case we've lost an SMP event */
1148         wake_up(&gator_buffer_wait);
1149
1150         return written > 0 ? written : -EFAULT;
1151 }
1152
1153 static const struct file_operations gator_event_buffer_fops = {
1154         .open = userspace_buffer_open,
1155         .release = userspace_buffer_release,
1156         .read = userspace_buffer_read,
1157 };
1158
1159 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1160 {
1161         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset);
1162 }
1163
1164 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1165 {
1166         unsigned long val;
1167         int retval;
1168
1169         if (*offset)
1170                 return -EINVAL;
1171
1172         retval = gatorfs_ulong_from_user(&val, buf, count);
1173         if (retval)
1174                 return retval;
1175
1176         retval = gator_set_backtrace(val);
1177
1178         if (retval)
1179                 return retval;
1180         return count;
1181 }
1182
1183 static const struct file_operations depth_fops = {
1184         .read = depth_read,
1185         .write = depth_write
1186 };
1187
1188 static void gator_op_create_files(struct super_block *sb, struct dentry *root)
1189 {
1190         struct dentry *dir;
1191         struct gator_interface *gi;
1192         int cpu;
1193         int err;
1194
1195         /* reinitialize default values */
1196         gator_cpu_cores = 0;
1197         for_each_present_cpu(cpu) {
1198                 gator_cpu_cores++;
1199         }
1200         userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1201         gator_response_type = 1;
1202         gator_live_rate = 0;
1203
1204         gatorfs_create_file(sb, root, "enable", &enable_fops);
1205         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1206         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1207         gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1208         gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1209         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1210         gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1211         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1212         gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started);
1213         gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate);
1214
1215         gator_annotate_create_files(sb, root);
1216
1217         /* Linux Events */
1218         dir = gatorfs_mkdir(sb, root, "events");
1219         gator_pmu_create_files(sb, root, dir);
1220         list_for_each_entry(gi, &gator_events, list)
1221                 if (gi->create_files) {
1222                         err = gi->create_files(sb, dir);
1223                         if (err != 0)
1224                                 pr_err("gator: create_files failed for %s\n", gi->name);
1225                 }
1226
1227         /* Sched Events */
1228         sched_trace_create_files(sb, dir);
1229
1230         /* Power interface */
1231         gator_trace_power_create_files(sb, dir);
1232 }
1233
1234 /******************************************************************************
1235  * Module
1236  ******************************************************************************/
1237
1238 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0)
1239
1240 #define GATOR_TRACEPOINTS \
1241         GATOR_HANDLE_TRACEPOINT(block_rq_complete); \
1242         GATOR_HANDLE_TRACEPOINT(cpu_frequency); \
1243         GATOR_HANDLE_TRACEPOINT(cpu_idle); \
1244         GATOR_HANDLE_TRACEPOINT(cpu_migrate_begin); \
1245         GATOR_HANDLE_TRACEPOINT(cpu_migrate_current); \
1246         GATOR_HANDLE_TRACEPOINT(cpu_migrate_finish); \
1247         GATOR_HANDLE_TRACEPOINT(irq_handler_exit); \
1248         GATOR_HANDLE_TRACEPOINT(mali_hw_counter); \
1249         GATOR_HANDLE_TRACEPOINT(mali_job_slots_event); \
1250         GATOR_HANDLE_TRACEPOINT(mali_mmu_as_in_use); \
1251         GATOR_HANDLE_TRACEPOINT(mali_mmu_as_released); \
1252         GATOR_HANDLE_TRACEPOINT(mali_page_fault_insert_pages); \
1253         GATOR_HANDLE_TRACEPOINT(mali_pm_status); \
1254         GATOR_HANDLE_TRACEPOINT(mali_sw_counter); \
1255         GATOR_HANDLE_TRACEPOINT(mali_sw_counters); \
1256         GATOR_HANDLE_TRACEPOINT(mali_timeline_event); \
1257         GATOR_HANDLE_TRACEPOINT(mali_total_alloc_pages_change); \
1258         GATOR_HANDLE_TRACEPOINT(mm_page_alloc); \
1259         GATOR_HANDLE_TRACEPOINT(mm_page_free); \
1260         GATOR_HANDLE_TRACEPOINT(mm_page_free_batched); \
1261         GATOR_HANDLE_TRACEPOINT(sched_process_exec); \
1262         GATOR_HANDLE_TRACEPOINT(sched_process_fork); \
1263         GATOR_HANDLE_TRACEPOINT(sched_process_free); \
1264         GATOR_HANDLE_TRACEPOINT(sched_switch); \
1265         GATOR_HANDLE_TRACEPOINT(softirq_exit); \
1266         GATOR_HANDLE_TRACEPOINT(task_rename); \
1267
1268 #define GATOR_HANDLE_TRACEPOINT(probe_name) \
1269         struct tracepoint *gator_tracepoint_##probe_name
1270 GATOR_TRACEPOINTS;
1271 #undef GATOR_HANDLE_TRACEPOINT
1272
1273 static void gator_save_tracepoint(struct tracepoint *tp, void *priv)
1274 {
1275 #define GATOR_HANDLE_TRACEPOINT(probe_name) \
1276         do { \
1277                 if (strcmp(tp->name, #probe_name) == 0) { \
1278                         gator_tracepoint_##probe_name = tp; \
1279                         return; \
1280                 } \
1281         } while (0)
1282 GATOR_TRACEPOINTS;
1283 #undef GATOR_HANDLE_TRACEPOINT
1284 }
1285
1286 #else
1287
1288 #define for_each_kernel_tracepoint(fct, priv)
1289
1290 #endif
1291
1292 static int __init gator_module_init(void)
1293 {
1294         for_each_kernel_tracepoint(gator_save_tracepoint, NULL);
1295
1296         if (gatorfs_register())
1297                 return -1;
1298
1299         if (gator_init()) {
1300                 gatorfs_unregister();
1301                 return -1;
1302         }
1303
1304         setup_deferrable_timer_on_stack(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1305
1306         /* Initialize the list of cpuids */
1307         memset(gator_cpuids, -1, sizeof(gator_cpuids));
1308         on_each_cpu(gator_read_cpuid, NULL, 1);
1309
1310         return 0;
1311 }
1312
1313 static void __exit gator_module_exit(void)
1314 {
1315         del_timer_sync(&gator_buffer_wake_up_timer);
1316         tracepoint_synchronize_unregister();
1317         gator_exit();
1318         gatorfs_unregister();
1319         gator_pmu_exit();
1320 }
1321
1322 module_init(gator_module_init);
1323 module_exit(gator_module_exit);
1324
1325 MODULE_LICENSE("GPL");
1326 MODULE_AUTHOR("ARM Ltd");
1327 MODULE_DESCRIPTION("Gator system profiler");
1328 #define STRIFY2(ARG) #ARG
1329 #define STRIFY(ARG) STRIFY2(ARG)
1330 MODULE_VERSION(STRIFY(PROTOCOL_VERSION));