include/linux/perf_event.h

   1 /*
   2  * Performance events:
   3  *
   4  *    Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de>
   5  *    Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar
   6  *    Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra
   7  *
   8  * Data type definitions, declarations, prototypes.
   9  *
  10  *    Started by: Thomas Gleixner and Ingo Molnar
  11  *
  12  * For licencing details see kernel-base/COPYING
  13  */
  14 #ifndef _LINUX_PERF_EVENT_H
  15 #define _LINUX_PERF_EVENT_H
  16
  17 #include <linux/types.h>
  18 #include <linux/ioctl.h>
  19 #include <asm/byteorder.h>
  20
  21 /*
  22  * User-space ABI bits:
  23  */
  24
  25 /*
  26  * attr.type
  27  */
  28 enum perf_type_id {
  29         PERF_TYPE_HARDWARE                      = 0,
  30         PERF_TYPE_SOFTWARE                      = 1,
  31         PERF_TYPE_TRACEPOINT                    = 2,
  32         PERF_TYPE_HW_CACHE                      = 3,
  33         PERF_TYPE_RAW                           = 4,
  34         PERF_TYPE_BREAKPOINT                    = 5,
  35
  36         PERF_TYPE_MAX,                          /* non-ABI */
  37 };
  38
  39 /*
  40  * Generalized performance event event_id types, used by the
  41  * attr.event_id parameter of the sys_perf_event_open()
  42  * syscall:
  43  */
  44 enum perf_hw_id {
  45         /*
  46          * Common hardware events, generalized by the kernel:
  47          */
  48         PERF_COUNT_HW_CPU_CYCLES                = 0,
  49         PERF_COUNT_HW_INSTRUCTIONS              = 1,
  50         PERF_COUNT_HW_CACHE_REFERENCES          = 2,
  51         PERF_COUNT_HW_CACHE_MISSES              = 3,
  52         PERF_COUNT_HW_BRANCH_INSTRUCTIONS       = 4,
  53         PERF_COUNT_HW_BRANCH_MISSES             = 5,
  54         PERF_COUNT_HW_BUS_CYCLES                = 6,
  55         PERF_COUNT_HW_STALLED_CYCLES_FRONTEND   = 7,
  56         PERF_COUNT_HW_STALLED_CYCLES_BACKEND    = 8,
  57         PERF_COUNT_HW_REF_CPU_CYCLES            = 9,
  58
  59         PERF_COUNT_HW_MAX,                      /* non-ABI */
  60 };
  61
  62 /*
  63  * Generalized hardware cache events:
  64  *
  65  *       { L1-D, L1-I, LLC, ITLB, DTLB, BPU, NODE } x
  66  *       { read, write, prefetch } x
  67  *       { accesses, misses }
  68  */
  69 enum perf_hw_cache_id {
  70         PERF_COUNT_HW_CACHE_L1D                 = 0,
  71         PERF_COUNT_HW_CACHE_L1I                 = 1,
  72         PERF_COUNT_HW_CACHE_LL                  = 2,
  73         PERF_COUNT_HW_CACHE_DTLB                = 3,
  74         PERF_COUNT_HW_CACHE_ITLB                = 4,
  75         PERF_COUNT_HW_CACHE_BPU                 = 5,
  76         PERF_COUNT_HW_CACHE_NODE                = 6,
  77
  78         PERF_COUNT_HW_CACHE_MAX,                /* non-ABI */
  79 };
  80
  81 enum perf_hw_cache_op_id {
  82         PERF_COUNT_HW_CACHE_OP_READ             = 0,
  83         PERF_COUNT_HW_CACHE_OP_WRITE            = 1,
  84         PERF_COUNT_HW_CACHE_OP_PREFETCH         = 2,
  85
  86         PERF_COUNT_HW_CACHE_OP_MAX,             /* non-ABI */
  87 };
  88
  89 enum perf_hw_cache_op_result_id {
  90         PERF_COUNT_HW_CACHE_RESULT_ACCESS       = 0,
  91         PERF_COUNT_HW_CACHE_RESULT_MISS         = 1,
  92
  93         PERF_COUNT_HW_CACHE_RESULT_MAX,         /* non-ABI */
  94 };
  95
  96 /*
  97  * Special "software" events provided by the kernel, even if the hardware
  98  * does not support performance events. These events measure various
  99  * physical and sw events of the kernel (and allow the profiling of them as
 100  * well):
 101  */
 102 enum perf_sw_ids {
 103         PERF_COUNT_SW_CPU_CLOCK                 = 0,
 104         PERF_COUNT_SW_TASK_CLOCK                = 1,
 105         PERF_COUNT_SW_PAGE_FAULTS               = 2,
 106         PERF_COUNT_SW_CONTEXT_SWITCHES          = 3,
 107         PERF_COUNT_SW_CPU_MIGRATIONS            = 4,
 108         PERF_COUNT_SW_PAGE_FAULTS_MIN           = 5,
 109         PERF_COUNT_SW_PAGE_FAULTS_MAJ           = 6,
 110         PERF_COUNT_SW_ALIGNMENT_FAULTS          = 7,
 111         PERF_COUNT_SW_EMULATION_FAULTS          = 8,
 112
 113         PERF_COUNT_SW_MAX,                      /* non-ABI */
 114 };
 115
 116 /*
 117  * Bits that can be set in attr.sample_type to request information
 118  * in the overflow packets.
 119  */
 120 enum perf_event_sample_format {
 121         PERF_SAMPLE_IP                          = 1U << 0,
 122         PERF_SAMPLE_TID                         = 1U << 1,
 123         PERF_SAMPLE_TIME                        = 1U << 2,
 124         PERF_SAMPLE_ADDR                        = 1U << 3,
 125         PERF_SAMPLE_READ                        = 1U << 4,
 126         PERF_SAMPLE_CALLCHAIN                   = 1U << 5,
 127         PERF_SAMPLE_ID                          = 1U << 6,
 128         PERF_SAMPLE_CPU                         = 1U << 7,
 129         PERF_SAMPLE_PERIOD                      = 1U << 8,
 130         PERF_SAMPLE_STREAM_ID                   = 1U << 9,
 131         PERF_SAMPLE_RAW                         = 1U << 10,
 132         PERF_SAMPLE_BRANCH_STACK                = 1U << 11,
 133         PERF_SAMPLE_REGS_USER                   = 1U << 12,
 134         PERF_SAMPLE_STACK_USER                  = 1U << 13,
 135
 136         PERF_SAMPLE_MAX = 1U << 14,             /* non-ABI */
 137 };
 138
 139 /*
 140  * values to program into branch_sample_type when PERF_SAMPLE_BRANCH is set
 141  *
 142  * If the user does not pass priv level information via branch_sample_type,
 143  * the kernel uses the event's priv level. Branch and event priv levels do
 144  * not have to match. Branch priv level is checked for permissions.
 145  *
 146  * The branch types can be combined, however BRANCH_ANY covers all types
 147  * of branches and therefore it supersedes all the other types.
 148  */
 149 enum perf_branch_sample_type {
 150         PERF_SAMPLE_BRANCH_USER         = 1U << 0, /* user branches */
 151         PERF_SAMPLE_BRANCH_KERNEL       = 1U << 1, /* kernel branches */
 152         PERF_SAMPLE_BRANCH_HV           = 1U << 2, /* hypervisor branches */
 153
 154         PERF_SAMPLE_BRANCH_ANY          = 1U << 3, /* any branch types */
 155         PERF_SAMPLE_BRANCH_ANY_CALL     = 1U << 4, /* any call branch */
 156         PERF_SAMPLE_BRANCH_ANY_RETURN   = 1U << 5, /* any return branch */
 157         PERF_SAMPLE_BRANCH_IND_CALL     = 1U << 6, /* indirect calls */
 158
 159         PERF_SAMPLE_BRANCH_MAX          = 1U << 7, /* non-ABI */
 160 };
 161
 162 #define PERF_SAMPLE_BRANCH_PLM_ALL \
 163         (PERF_SAMPLE_BRANCH_USER|\
 164          PERF_SAMPLE_BRANCH_KERNEL|\
 165          PERF_SAMPLE_BRANCH_HV)
 166
 167 /*
 168  * Values to determine ABI of the registers dump.
 169  */
 170 enum perf_sample_regs_abi {
 171         PERF_SAMPLE_REGS_ABI_NONE       = 0,
 172         PERF_SAMPLE_REGS_ABI_32         = 1,
 173         PERF_SAMPLE_REGS_ABI_64         = 2,
 174 };
 175
 176 /*
 177  * The format of the data returned by read() on a perf event fd,
 178  * as specified by attr.read_format:
 179  *
 180  * struct read_format {
 181  *      { u64           value;
 182  *        { u64         time_enabled; } && PERF_FORMAT_TOTAL_TIME_ENABLED
 183  *        { u64         time_running; } && PERF_FORMAT_TOTAL_TIME_RUNNING
 184  *        { u64         id;           } && PERF_FORMAT_ID
 185  *      } && !PERF_FORMAT_GROUP
 186  *
 187  *      { u64           nr;
 188  *        { u64         time_enabled; } && PERF_FORMAT_TOTAL_TIME_ENABLED
 189  *        { u64         time_running; } && PERF_FORMAT_TOTAL_TIME_RUNNING
 190  *        { u64         value;
 191  *          { u64       id;           } && PERF_FORMAT_ID
 192  *        }             cntr[nr];
 193  *      } && PERF_FORMAT_GROUP
 194  * };
 195  */
 196 enum perf_event_read_format {
 197         PERF_FORMAT_TOTAL_TIME_ENABLED          = 1U << 0,
 198         PERF_FORMAT_TOTAL_TIME_RUNNING          = 1U << 1,
 199         PERF_FORMAT_ID                          = 1U << 2,
 200         PERF_FORMAT_GROUP                       = 1U << 3,
 201
 202         PERF_FORMAT_MAX = 1U << 4,              /* non-ABI */
 203 };
 204
 205 #define PERF_ATTR_SIZE_VER0     64      /* sizeof first published struct */
 206 #define PERF_ATTR_SIZE_VER1     72      /* add: config2 */
 207 #define PERF_ATTR_SIZE_VER2     80      /* add: branch_sample_type */
 208 #define PERF_ATTR_SIZE_VER3     96      /* add: sample_regs_user */
 209                                         /* add: sample_stack_user */
 210
 211 /*
 212  * Hardware event_id to monitor via a performance monitoring event:
 213  */
 214 struct perf_event_attr {
 215
 216         /*
 217          * Major type: hardware/software/tracepoint/etc.
 218          */
 219         __u32                   type;
 220
 221         /*
 222          * Size of the attr structure, for fwd/bwd compat.
 223          */
 224         __u32                   size;
 225
 226         /*
 227          * Type specific configuration information.
 228          */
 229         __u64                   config;
 230
 231         union {
 232                 __u64           sample_period;
 233                 __u64           sample_freq;
 234         };
 235
 236         __u64                   sample_type;
 237         __u64                   read_format;
 238
 239         __u64                   disabled       :  1, /* off by default        */
 240                                 inherit        :  1, /* children inherit it   */
 241                                 pinned         :  1, /* must always be on PMU */
 242                                 exclusive      :  1, /* only group on PMU     */
 243                                 exclude_user   :  1, /* don't count user      */
 244                                 exclude_kernel :  1, /* ditto kernel          */
 245                                 exclude_hv     :  1, /* ditto hypervisor      */
 246                                 exclude_idle   :  1, /* don't count when idle */
 247                                 mmap           :  1, /* include mmap data     */
 248                                 comm           :  1, /* include comm data     */
 249                                 freq           :  1, /* use freq, not period  */
 250                                 inherit_stat   :  1, /* per task counts       */
 251                                 enable_on_exec :  1, /* next exec enables     */
 252                                 task           :  1, /* trace fork/exit       */
 253                                 watermark      :  1, /* wakeup_watermark      */
 254                                 /*
 255                                  * precise_ip:
 256                                  *
 257                                  *  0 - SAMPLE_IP can have arbitrary skid
 258                                  *  1 - SAMPLE_IP must have constant skid
 259                                  *  2 - SAMPLE_IP requested to have 0 skid
 260                                  *  3 - SAMPLE_IP must have 0 skid
 261                                  *
 262                                  *  See also PERF_RECORD_MISC_EXACT_IP
 263                                  */
 264                                 precise_ip     :  2, /* skid constraint       */
 265                                 mmap_data      :  1, /* non-exec mmap data    */
 266                                 sample_id_all  :  1, /* sample_type all events */
 267
 268                                 exclude_host   :  1, /* don't count in host   */
 269                                 exclude_guest  :  1, /* don't count in guest  */
 270
 271                                 exclude_callchain_kernel : 1, /* exclude kernel callchains */
 272                                 exclude_callchain_user   : 1, /* exclude user callchains */
 273
 274                                 __reserved_1   : 41;
 275
 276         union {
 277                 __u32           wakeup_events;    /* wakeup every n events */
 278                 __u32           wakeup_watermark; /* bytes before wakeup   */
 279         };
 280
 281         __u32                   bp_type;
 282         union {
 283                 __u64           bp_addr;
 284                 __u64           config1; /* extension of config */
 285         };
 286         union {
 287                 __u64           bp_len;
 288                 __u64           config2; /* extension of config1 */
 289         };
 290         __u64   branch_sample_type; /* enum perf_branch_sample_type */
 291
 292         /*
 293          * Defines set of user regs to dump on samples.
 294          * See asm/perf_regs.h for details.
 295          */
 296         __u64   sample_regs_user;
 297
 298         /*
 299          * Defines size of the user stack to dump on samples.
 300          */
 301         __u32   sample_stack_user;
 302
 303         /* Align to u64. */
 304         __u32   __reserved_2;
 305 };
 306
 307 #define perf_flags(attr)        (*(&(attr)->read_format + 1))
 308
 309 /*
 310  * Ioctls that can be done on a perf event fd:
 311  */
 312 #define PERF_EVENT_IOC_ENABLE           _IO ('$', 0)
 313 #define PERF_EVENT_IOC_DISABLE          _IO ('$', 1)
 314 #define PERF_EVENT_IOC_REFRESH          _IO ('$', 2)
 315 #define PERF_EVENT_IOC_RESET            _IO ('$', 3)
 316 #define PERF_EVENT_IOC_PERIOD           _IOW('$', 4, __u64)
 317 #define PERF_EVENT_IOC_SET_OUTPUT       _IO ('$', 5)
 318 #define PERF_EVENT_IOC_SET_FILTER       _IOW('$', 6, char *)
 319
 320 enum perf_event_ioc_flags {
 321         PERF_IOC_FLAG_GROUP             = 1U << 0,
 322 };
 323
 324 /*
 325  * Structure of the page that can be mapped via mmap
 326  */
 327 struct perf_event_mmap_page {
 328         __u32   version;                /* version number of this structure */
 329         __u32   compat_version;         /* lowest version this is compat with */
 330
 331         /*
 332          * Bits needed to read the hw events in user-space.
 333          *
 334          *   u32 seq, time_mult, time_shift, idx, width;
 335          *   u64 count, enabled, running;
 336          *   u64 cyc, time_offset;
 337          *   s64 pmc = 0;
 338          *
 339          *   do {
 340          *     seq = pc->lock;
 341          *     barrier()
 342          *
 343          *     enabled = pc->time_enabled;
 344          *     running = pc->time_running;
 345          *
 346          *     if (pc->cap_usr_time && enabled != running) {
 347          *       cyc = rdtsc();
 348          *       time_offset = pc->time_offset;
 349          *       time_mult   = pc->time_mult;
 350          *       time_shift  = pc->time_shift;
 351          *     }
 352          *
 353          *     idx = pc->index;
 354          *     count = pc->offset;
 355          *     if (pc->cap_usr_rdpmc && idx) {
 356          *       width = pc->pmc_width;
 357          *       pmc = rdpmc(idx - 1);
 358          *     }
 359          *
 360          *     barrier();
 361          *   } while (pc->lock != seq);
 362          *
 363          * NOTE: for obvious reason this only works on self-monitoring
 364          *       processes.
 365          */
 366         __u32   lock;                   /* seqlock for synchronization */
 367         __u32   index;                  /* hardware event identifier */
 368         __s64   offset;                 /* add to hardware event value */
 369         __u64   time_enabled;           /* time event active */
 370         __u64   time_running;           /* time event on cpu */
 371         union {
 372                 __u64   capabilities;
 373                 __u64   cap_usr_time  : 1,
 374                         cap_usr_rdpmc : 1,
 375                         cap_____res   : 62;
 376         };
 377
 378         /*
 379          * If cap_usr_rdpmc this field provides the bit-width of the value
 380          * read using the rdpmc() or equivalent instruction. This can be used
 381          * to sign extend the result like:
 382          *
 383          *   pmc <<= 64 - width;
 384          *   pmc >>= 64 - width; // signed shift right
 385          *   count += pmc;
 386          */
 387         __u16   pmc_width;
 388
 389         /*
 390          * If cap_usr_time the below fields can be used to compute the time
 391          * delta since time_enabled (in ns) using rdtsc or similar.
 392          *
 393          *   u64 quot, rem;
 394          *   u64 delta;
 395          *
 396          *   quot = (cyc >> time_shift);
 397          *   rem = cyc & ((1 << time_shift) - 1);
 398          *   delta = time_offset + quot * time_mult +
 399          *              ((rem * time_mult) >> time_shift);
 400          *
 401          * Where time_offset,time_mult,time_shift and cyc are read in the
 402          * seqcount loop described above. This delta can then be added to
 403          * enabled and possible running (if idx), improving the scaling:
 404          *
 405          *   enabled += delta;
 406          *   if (idx)
 407          *     running += delta;
 408          *
 409          *   quot = count / running;
 410          *   rem  = count % running;
 411          *   count = quot * enabled + (rem * enabled) / running;
 412          */
 413         __u16   time_shift;
 414         __u32   time_mult;
 415         __u64   time_offset;
 416
 417                 /*
 418                  * Hole for extension of the self monitor capabilities
 419                  */
 420
 421         __u64   __reserved[120];        /* align to 1k */
 422
 423         /*
 424          * Control data for the mmap() data buffer.
 425          *
 426          * User-space reading the @data_head value should issue an rmb(), on
 427          * SMP capable platforms, after reading this value -- see
 428          * perf_event_wakeup().
 429          *
 430          * When the mapping is PROT_WRITE the @data_tail value should be
 431          * written by userspace to reflect the last read data. In this case
 432          * the kernel will not over-write unread data.
 433          */
 434         __u64   data_head;              /* head in the data section */
 435         __u64   data_tail;              /* user-space written tail */
 436 };
 437
 438 #define PERF_RECORD_MISC_CPUMODE_MASK           (7 << 0)
 439 #define PERF_RECORD_MISC_CPUMODE_UNKNOWN        (0 << 0)
 440 #define PERF_RECORD_MISC_KERNEL                 (1 << 0)
 441 #define PERF_RECORD_MISC_USER                   (2 << 0)
 442 #define PERF_RECORD_MISC_HYPERVISOR             (3 << 0)
 443 #define PERF_RECORD_MISC_GUEST_KERNEL           (4 << 0)
 444 #define PERF_RECORD_MISC_GUEST_USER             (5 << 0)
 445
 446 /*
 447  * Indicates that the content of PERF_SAMPLE_IP points to
 448  * the actual instruction that triggered the event. See also
 449  * perf_event_attr::precise_ip.
 450  */
 451 #define PERF_RECORD_MISC_EXACT_IP               (1 << 14)
 452 /*
 453  * Reserve the last bit to indicate some extended misc field
 454  */
 455 #define PERF_RECORD_MISC_EXT_RESERVED           (1 << 15)
 456
 457 struct perf_event_header {
 458         __u32   type;
 459         __u16   misc;
 460         __u16   size;
 461 };
 462
 463 enum perf_event_type {
 464
 465         /*
 466          * If perf_event_attr.sample_id_all is set then all event types will
 467          * have the sample_type selected fields related to where/when
 468          * (identity) an event took place (TID, TIME, ID, CPU, STREAM_ID)
 469          * described in PERF_RECORD_SAMPLE below, it will be stashed just after
 470          * the perf_event_header and the fields already present for the existing
 471          * fields, i.e. at the end of the payload. That way a newer perf.data
 472          * file will be supported by older perf tools, with these new optional
 473          * fields being ignored.
 474          *
 475          * The MMAP events record the PROT_EXEC mappings so that we can
 476          * correlate userspace IPs to code. They have the following structure:
 477          *
 478          * struct {
 479          *      struct perf_event_header        header;
 480          *
 481          *      u32                             pid, tid;
 482          *      u64                             addr;
 483          *      u64                             len;
 484          *      u64                             pgoff;
 485          *      char                            filename[];
 486          * };
 487          */
 488         PERF_RECORD_MMAP                        = 1,
 489
 490         /*
 491          * struct {
 492          *      struct perf_event_header        header;
 493          *      u64                             id;
 494          *      u64                             lost;
 495          * };
 496          */
 497         PERF_RECORD_LOST                        = 2,
 498
 499         /*
 500          * struct {
 501          *      struct perf_event_header        header;
 502          *
 503          *      u32                             pid, tid;
 504          *      char                            comm[];
 505          * };
 506          */
 507         PERF_RECORD_COMM                        = 3,
 508
 509         /*
 510          * struct {
 511          *      struct perf_event_header        header;
 512          *      u32                             pid, ppid;
 513          *      u32                             tid, ptid;
 514          *      u64                             time;
 515          * };
 516          */
 517         PERF_RECORD_EXIT                        = 4,
 518
 519         /*
 520          * struct {
 521          *      struct perf_event_header        header;
 522          *      u64                             time;
 523          *      u64                             id;
 524          *      u64                             stream_id;
 525          * };
 526          */
 527         PERF_RECORD_THROTTLE                    = 5,
 528         PERF_RECORD_UNTHROTTLE                  = 6,
 529
 530         /*
 531          * struct {
 532          *      struct perf_event_header        header;
 533          *      u32                             pid, ppid;
 534          *      u32                             tid, ptid;
 535          *      u64                             time;
 536          * };
 537          */
 538         PERF_RECORD_FORK                        = 7,
 539
 540         /*
 541          * struct {
 542          *      struct perf_event_header        header;
 543          *      u32                             pid, tid;
 544          *
 545          *      struct read_format              values;
 546          * };
 547          */
 548         PERF_RECORD_READ                        = 8,
 549
 550         /*
 551          * struct {
 552          *      struct perf_event_header        header;
 553          *
 554          *      { u64                   ip;       } && PERF_SAMPLE_IP
 555          *      { u32                   pid, tid; } && PERF_SAMPLE_TID
 556          *      { u64                   time;     } && PERF_SAMPLE_TIME
 557          *      { u64                   addr;     } && PERF_SAMPLE_ADDR
 558          *      { u64                   id;       } && PERF_SAMPLE_ID
 559          *      { u64                   stream_id;} && PERF_SAMPLE_STREAM_ID
 560          *      { u32                   cpu, res; } && PERF_SAMPLE_CPU
 561          *      { u64                   period;   } && PERF_SAMPLE_PERIOD
 562          *
 563          *      { struct read_format    values;   } && PERF_SAMPLE_READ
 564          *
 565          *      { u64                   nr,
 566          *        u64                   ips[nr];  } && PERF_SAMPLE_CALLCHAIN
 567          *
 568          *      #
 569          *      # The RAW record below is opaque data wrt the ABI
 570          *      #
 571          *      # That is, the ABI doesn't make any promises wrt to
 572          *      # the stability of its content, it may vary depending
 573          *      # on event, hardware, kernel version and phase of
 574          *      # the moon.
 575          *      #
 576          *      # In other words, PERF_SAMPLE_RAW contents are not an ABI.
 577          *      #
 578          *
 579          *      { u32                   size;
 580          *        char                  data[size];}&& PERF_SAMPLE_RAW
 581          *
 582          *      { u64 from, to, flags } lbr[nr];} && PERF_SAMPLE_BRANCH_STACK
 583          *
 584          *      { u64                   abi; # enum perf_sample_regs_abi
 585          *        u64                   regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER
 586          *
 587          *      { u64                   size;
 588          *        char                  data[size];
 589          *        u64                   dyn_size; } && PERF_SAMPLE_STACK_USER
 590          * };
 591          */
 592         PERF_RECORD_SAMPLE                      = 9,
 593
 594         PERF_RECORD_MAX,                        /* non-ABI */
 595 };
 596
 597 #define PERF_MAX_STACK_DEPTH            127
 598
 599 enum perf_callchain_context {
 600         PERF_CONTEXT_HV                 = (__u64)-32,
 601         PERF_CONTEXT_KERNEL             = (__u64)-128,
 602         PERF_CONTEXT_USER               = (__u64)-512,
 603
 604         PERF_CONTEXT_GUEST              = (__u64)-2048,
 605         PERF_CONTEXT_GUEST_KERNEL       = (__u64)-2176,
 606         PERF_CONTEXT_GUEST_USER         = (__u64)-2560,
 607
 608         PERF_CONTEXT_MAX                = (__u64)-4095,
 609 };
 610
 611 #define PERF_FLAG_FD_NO_GROUP           (1U << 0)
 612 #define PERF_FLAG_FD_OUTPUT             (1U << 1)
 613 #define PERF_FLAG_PID_CGROUP            (1U << 2) /* pid=cgroup id, per-cpu mode only */
 614
 615 #ifdef __KERNEL__
 616 /*
 617  * Kernel-internal data types and definitions:
 618  */
 619
 620 #ifdef CONFIG_PERF_EVENTS
 621 # include <linux/cgroup.h>
 622 # include <asm/perf_event.h>
 623 # include <asm/local64.h>
 624 #endif
 625
 626 struct perf_guest_info_callbacks {
 627         int                             (*is_in_guest)(void);
 628         int                             (*is_user_mode)(void);
 629         unsigned long                   (*get_guest_ip)(void);
 630 };
 631
 632 #ifdef CONFIG_HAVE_HW_BREAKPOINT
 633 #include <asm/hw_breakpoint.h>
 634 #endif
 635
 636 #include <linux/list.h>
 637 #include <linux/mutex.h>
 638 #include <linux/rculist.h>
 639 #include <linux/rcupdate.h>
 640 #include <linux/spinlock.h>
 641 #include <linux/hrtimer.h>
 642 #include <linux/fs.h>
 643 #include <linux/pid_namespace.h>
 644 #include <linux/workqueue.h>
 645 #include <linux/ftrace.h>
 646 #include <linux/cpu.h>
 647 #include <linux/irq_work.h>
 648 #include <linux/static_key.h>
 649 #include <linux/atomic.h>
 650 #include <linux/sysfs.h>
 651 #include <linux/perf_regs.h>
 652 #include <asm/local.h>
 653
 654 struct perf_callchain_entry {
 655         __u64                           nr;
 656         __u64                           ip[PERF_MAX_STACK_DEPTH];
 657 };
 658
 659 struct perf_raw_record {
 660         u32                             size;
 661         void                            *data;
 662 };
 663
 664 /*
 665  * single taken branch record layout:
 666  *
 667  *      from: source instruction (may not always be a branch insn)
 668  *        to: branch target
 669  *   mispred: branch target was mispredicted
 670  * predicted: branch target was predicted
 671  *
 672  * support for mispred, predicted is optional. In case it
 673  * is not supported mispred = predicted = 0.
 674  */
 675 struct perf_branch_entry {
 676         __u64   from;
 677         __u64   to;
 678         __u64   mispred:1,  /* target mispredicted */
 679                 predicted:1,/* target predicted */
 680                 reserved:62;
 681 };
 682
 683 /*
 684  * branch stack layout:
 685  *  nr: number of taken branches stored in entries[]
 686  *
 687  * Note that nr can vary from sample to sample
 688  * branches (to, from) are stored from most recent
 689  * to least recent, i.e., entries[0] contains the most
 690  * recent branch.
 691  */
 692 struct perf_branch_stack {
 693         __u64                           nr;
 694         struct perf_branch_entry        entries[0];
 695 };
 696
 697 struct perf_regs_user {
 698         __u64           abi;
 699         struct pt_regs  *regs;
 700 };
 701
 702 struct task_struct;
 703
 704 /*
 705  * extra PMU register associated with an event
 706  */
 707 struct hw_perf_event_extra {
 708         u64             config; /* register value */
 709         unsigned int    reg;    /* register address or index */
 710         int             alloc;  /* extra register already allocated */
 711         int             idx;    /* index in shared_regs->regs[] */
 712 };
 713
 714 /**
 715  * struct hw_perf_event - performance event hardware details:
 716  */
 717 struct hw_perf_event {
 718 #ifdef CONFIG_PERF_EVENTS
 719         union {
 720                 struct { /* hardware */
 721                         u64             config;
 722                         u64             last_tag;
 723                         unsigned long   config_base;
 724                         unsigned long   event_base;
 725                         int             event_base_rdpmc;
 726                         int             idx;
 727                         int             last_cpu;
 728
 729                         struct hw_perf_event_extra extra_reg;
 730                         struct hw_perf_event_extra branch_reg;
 731                 };
 732                 struct { /* software */
 733                         struct hrtimer  hrtimer;
 734                 };
 735 #ifdef CONFIG_HAVE_HW_BREAKPOINT
 736                 struct { /* breakpoint */
 737                         struct arch_hw_breakpoint       info;
 738                         struct list_head                bp_list;
 739                         /*
 740                          * Crufty hack to avoid the chicken and egg
 741                          * problem hw_breakpoint has with context
 742                          * creation and event initalization.
 743                          */
 744                         struct task_struct              *bp_target;
 745                 };
 746 #endif
 747         };
 748         int                             state;
 749         local64_t                       prev_count;
 750         u64                             sample_period;
 751         u64                             last_period;
 752         local64_t                       period_left;
 753         u64                             interrupts_seq;
 754         u64                             interrupts;
 755
 756         u64                             freq_time_stamp;
 757         u64                             freq_count_stamp;
 758 #endif
 759 };
 760
 761 /*
 762  * hw_perf_event::state flags
 763  */
 764 #define PERF_HES_STOPPED        0x01 /* the counter is stopped */
 765 #define PERF_HES_UPTODATE       0x02 /* event->count up-to-date */
 766 #define PERF_HES_ARCH           0x04
 767
 768 struct perf_event;
 769
 770 /*
 771  * Common implementation detail of pmu::{start,commit,cancel}_txn
 772  */
 773 #define PERF_EVENT_TXN 0x1
 774
 775 /**
 776  * struct pmu - generic performance monitoring unit
 777  */
 778 struct pmu {
 779         struct list_head                entry;
 780
 781         struct device                   *dev;
 782         const struct attribute_group    **attr_groups;
 783         char                            *name;
 784         int                             type;
 785
 786         int * __percpu                  pmu_disable_count;
 787         struct perf_cpu_context * __percpu pmu_cpu_context;
 788         int                             task_ctx_nr;
 789
 790         /*
 791          * Fully disable/enable this PMU, can be used to protect from the PMI
 792          * as well as for lazy/batch writing of the MSRs.
 793          */
 794         void (*pmu_enable)              (struct pmu *pmu); /* optional */
 795         void (*pmu_disable)             (struct pmu *pmu); /* optional */
 796
 797         /*
 798          * Try and initialize the event for this PMU.
 799          * Should return -ENOENT when the @event doesn't match this PMU.
 800          */
 801         int (*event_init)               (struct perf_event *event);
 802
 803 #define PERF_EF_START   0x01            /* start the counter when adding    */
 804 #define PERF_EF_RELOAD  0x02            /* reload the counter when starting */
 805 #define PERF_EF_UPDATE  0x04            /* update the counter when stopping */
 806
 807         /*
 808          * Adds/Removes a counter to/from the PMU, can be done inside
 809          * a transaction, see the ->*_txn() methods.
 810          */
 811         int  (*add)                     (struct perf_event *event, int flags);
 812         void (*del)                     (struct perf_event *event, int flags);
 813
 814         /*
 815          * Starts/Stops a counter present on the PMU. The PMI handler
 816          * should stop the counter when perf_event_overflow() returns
 817          * !0. ->start() will be used to continue.
 818          */
 819         void (*start)                   (struct perf_event *event, int flags);
 820         void (*stop)                    (struct perf_event *event, int flags);
 821
 822         /*
 823          * Updates the counter value of the event.
 824          */
 825         void (*read)                    (struct perf_event *event);
 826
 827         /*
 828          * Group events scheduling is treated as a transaction, add
 829          * group events as a whole and perform one schedulability test.
 830          * If the test fails, roll back the whole group
 831          *
 832          * Start the transaction, after this ->add() doesn't need to
 833          * do schedulability tests.
 834          */
 835         void (*start_txn)               (struct pmu *pmu); /* optional */
 836         /*
 837          * If ->start_txn() disabled the ->add() schedulability test
 838          * then ->commit_txn() is required to perform one. On success
 839          * the transaction is closed. On error the transaction is kept
 840          * open until ->cancel_txn() is called.
 841          */
 842         int  (*commit_txn)              (struct pmu *pmu); /* optional */
 843         /*
 844          * Will cancel the transaction, assumes ->del() is called
 845          * for each successful ->add() during the transaction.
 846          */
 847         void (*cancel_txn)              (struct pmu *pmu); /* optional */
 848
 849         /*
 850          * Will return the value for perf_event_mmap_page::index for this event,
 851          * if no implementation is provided it will default to: event->hw.idx + 1.
 852          */
 853         int (*event_idx)                (struct perf_event *event); /*optional */
 854
 855         /*
 856          * flush branch stack on context-switches (needed in cpu-wide mode)
 857          */
 858         void (*flush_branch_stack)      (void);
 859 };
 860
 861 /**
 862  * enum perf_event_active_state - the states of a event
 863  */
 864 enum perf_event_active_state {
 865         PERF_EVENT_STATE_ERROR          = -2,
 866         PERF_EVENT_STATE_OFF            = -1,
 867         PERF_EVENT_STATE_INACTIVE       =  0,
 868         PERF_EVENT_STATE_ACTIVE         =  1,
 869 };
 870
 871 struct file;
 872 struct perf_sample_data;
 873
 874 typedef void (*perf_overflow_handler_t)(struct perf_event *,
 875                                         struct perf_sample_data *,
 876                                         struct pt_regs *regs);
 877
 878 enum perf_group_flag {
 879         PERF_GROUP_SOFTWARE             = 0x1,
 880 };
 881
 882 #define SWEVENT_HLIST_BITS              8
 883 #define SWEVENT_HLIST_SIZE              (1 << SWEVENT_HLIST_BITS)
 884
 885 struct swevent_hlist {
 886         struct hlist_head               heads[SWEVENT_HLIST_SIZE];
 887         struct rcu_head                 rcu_head;
 888 };
 889
 890 #define PERF_ATTACH_CONTEXT     0x01
 891 #define PERF_ATTACH_GROUP       0x02
 892 #define PERF_ATTACH_TASK        0x04
 893
 894 #ifdef CONFIG_CGROUP_PERF
 895 /*
 896  * perf_cgroup_info keeps track of time_enabled for a cgroup.
 897  * This is a per-cpu dynamically allocated data structure.
 898  */
 899 struct perf_cgroup_info {
 900         u64                             time;
 901         u64                             timestamp;
 902 };
 903
 904 struct perf_cgroup {
 905         struct                          cgroup_subsys_state css;
 906         struct                          perf_cgroup_info *info; /* timing info, one per cpu */
 907 };
 908 #endif
 909
 910 struct ring_buffer;
 911
 912 /**
 913  * struct perf_event - performance event kernel representation:
 914  */
 915 struct perf_event {
 916 #ifdef CONFIG_PERF_EVENTS
 917         struct list_head                group_entry;
 918         struct list_head                event_entry;
 919         struct list_head                sibling_list;
 920         struct hlist_node               hlist_entry;
 921         int                             nr_siblings;
 922         int                             group_flags;
 923         struct perf_event               *group_leader;
 924         struct pmu                      *pmu;
 925
 926         enum perf_event_active_state    state;
 927         unsigned int                    attach_state;
 928         local64_t                       count;
 929         atomic64_t                      child_count;
 930
 931         /*
 932          * These are the total time in nanoseconds that the event
 933          * has been enabled (i.e. eligible to run, and the task has
 934          * been scheduled in, if this is a per-task event)
 935          * and running (scheduled onto the CPU), respectively.
 936          *
 937          * They are computed from tstamp_enabled, tstamp_running and
 938          * tstamp_stopped when the event is in INACTIVE or ACTIVE state.
 939          */
 940         u64                             total_time_enabled;
 941         u64                             total_time_running;
 942
 943         /*
 944          * These are timestamps used for computing total_time_enabled
 945          * and total_time_running when the event is in INACTIVE or
 946          * ACTIVE state, measured in nanoseconds from an arbitrary point
 947          * in time.
 948          * tstamp_enabled: the notional time when the event was enabled
 949          * tstamp_running: the notional time when the event was scheduled on
 950          * tstamp_stopped: in INACTIVE state, the notional time when the
 951          *      event was scheduled off.
 952          */
 953         u64                             tstamp_enabled;
 954         u64                             tstamp_running;
 955         u64                             tstamp_stopped;
 956
 957         /*
 958          * timestamp shadows the actual context timing but it can
 959          * be safely used in NMI interrupt context. It reflects the
 960          * context time as it was when the event was last scheduled in.
 961          *
 962          * ctx_time already accounts for ctx->timestamp. Therefore to
 963          * compute ctx_time for a sample, simply add perf_clock().
 964          */
 965         u64                             shadow_ctx_time;
 966
 967         struct perf_event_attr          attr;
 968         u16                             header_size;
 969         u16                             id_header_size;
 970         u16                             read_size;
 971         struct hw_perf_event            hw;
 972
 973         struct perf_event_context       *ctx;
 974         atomic_long_t                   refcount;
 975
 976         /*
 977          * These accumulate total time (in nanoseconds) that children
 978          * events have been enabled and running, respectively.
 979          */
 980         atomic64_t                      child_total_time_enabled;
 981         atomic64_t                      child_total_time_running;
 982
 983         /*
 984          * Protect attach/detach and child_list:
 985          */
 986         struct mutex                    child_mutex;
 987         struct list_head                child_list;
 988         struct perf_event               *parent;
 989
 990         int                             oncpu;
 991         int                             cpu;
 992
 993         struct list_head                owner_entry;
 994         struct task_struct              *owner;
 995
 996         /* mmap bits */
 997         struct mutex                    mmap_mutex;
 998         atomic_t                        mmap_count;
 999         int                             mmap_locked;
1000         struct user_struct              *mmap_user;
1001         struct ring_buffer              *rb;
1002         struct list_head                rb_entry;
1003
1004         /* poll related */
1005         wait_queue_head_t               waitq;
1006         struct fasync_struct            *fasync;
1007
1008         /* delayed work for NMIs and such */
1009         int                             pending_wakeup;
1010         int                             pending_kill;
1011         int                             pending_disable;
1012         struct irq_work                 pending;
1013
1014         atomic_t                        event_limit;
1015
1016         void (*destroy)(struct perf_event *);
1017         struct rcu_head                 rcu_head;
1018
1019         struct pid_namespace            *ns;
1020         u64                             id;
1021
1022         perf_overflow_handler_t         overflow_handler;
1023         void                            *overflow_handler_context;
1024
1025 #ifdef CONFIG_EVENT_TRACING
1026         struct ftrace_event_call        *tp_event;
1027         struct event_filter             *filter;
1028 #ifdef CONFIG_FUNCTION_TRACER
1029         struct ftrace_ops               ftrace_ops;
1030 #endif
1031 #endif
1032
1033 #ifdef CONFIG_CGROUP_PERF
1034         struct perf_cgroup              *cgrp; /* cgroup event is attach to */
1035         int                             cgrp_defer_enabled;
1036 #endif
1037
1038 #endif /* CONFIG_PERF_EVENTS */
1039 };
1040
1041 enum perf_event_context_type {
1042         task_context,
1043         cpu_context,
1044 };
1045
1046 /**
1047  * struct perf_event_context - event context structure
1048  *
1049  * Used as a container for task events and CPU events as well:
1050  */
1051 struct perf_event_context {
1052         struct pmu                      *pmu;
1053         enum perf_event_context_type    type;
1054         /*
1055          * Protect the states of the events in the list,
1056          * nr_active, and the list:
1057          */
1058         raw_spinlock_t                  lock;
1059         /*
1060          * Protect the list of events.  Locking either mutex or lock
1061          * is sufficient to ensure the list doesn't change; to change
1062          * the list you need to lock both the mutex and the spinlock.
1063          */
1064         struct mutex                    mutex;
1065
1066         struct list_head                pinned_groups;
1067         struct list_head                flexible_groups;
1068         struct list_head                event_list;
1069         int                             nr_events;
1070         int                             nr_active;
1071         int                             is_active;
1072         int                             nr_stat;
1073         int                             nr_freq;
1074         int                             rotate_disable;
1075         atomic_t                        refcount;
1076         struct task_struct              *task;
1077
1078         /*
1079          * Context clock, runs when context enabled.
1080          */
1081         u64                             time;
1082         u64                             timestamp;
1083
1084         /*
1085          * These fields let us detect when two contexts have both
1086          * been cloned (inherited) from a common ancestor.
1087          */
1088         struct perf_event_context       *parent_ctx;
1089         u64                             parent_gen;
1090         u64                             generation;
1091         int                             pin_count;
1092         int                             nr_cgroups;      /* cgroup evts */
1093         int                             nr_branch_stack; /* branch_stack evt */
1094         struct rcu_head                 rcu_head;
1095 };
1096
1097 /*
1098  * Number of contexts where an event can trigger:
1099  *      task, softirq, hardirq, nmi.
1100  */
1101 #define PERF_NR_CONTEXTS        4
1102
1103 /**
1104  * struct perf_event_cpu_context - per cpu event context structure
1105  */
1106 struct perf_cpu_context {
1107         struct perf_event_context       ctx;
1108         struct perf_event_context       *task_ctx;
1109         int                             active_oncpu;
1110         int                             exclusive;
1111         struct list_head                rotation_list;
1112         int                             jiffies_interval;
1113         struct pmu                      *active_pmu;
1114         struct perf_cgroup              *cgrp;
1115 };
1116
1117 struct perf_output_handle {
1118         struct perf_event               *event;
1119         struct ring_buffer              *rb;
1120         unsigned long                   wakeup;
1121         unsigned long                   size;
1122         void                            *addr;
1123         int                             page;
1124 };
1125
1126 #ifdef CONFIG_PERF_EVENTS
1127
1128 extern int perf_pmu_register(struct pmu *pmu, char *name, int type);
1129 extern void perf_pmu_unregister(struct pmu *pmu);
1130
1131 extern int perf_num_counters(void);
1132 extern const char *perf_pmu_name(void);
1133 extern void __perf_event_task_sched_in(struct task_struct *prev,
1134                                        struct task_struct *task);
1135 extern void __perf_event_task_sched_out(struct task_struct *prev,
1136                                         struct task_struct *next);
1137 extern int perf_event_init_task(struct task_struct *child);
1138 extern void perf_event_exit_task(struct task_struct *child);
1139 extern void perf_event_free_task(struct task_struct *task);
1140 extern void perf_event_delayed_put(struct task_struct *task);
1141 extern void perf_event_print_debug(void);
1142 extern void perf_pmu_disable(struct pmu *pmu);
1143 extern void perf_pmu_enable(struct pmu *pmu);
1144 extern int perf_event_task_disable(void);
1145 extern int perf_event_task_enable(void);
1146 extern int perf_event_refresh(struct perf_event *event, int refresh);
1147 extern void perf_event_update_userpage(struct perf_event *event);
1148 extern int perf_event_release_kernel(struct perf_event *event);
1149 extern struct perf_event *
1150 perf_event_create_kernel_counter(struct perf_event_attr *attr,
1151                                 int cpu,
1152                                 struct task_struct *task,
1153                                 perf_overflow_handler_t callback,
1154                                 void *context);
1155 extern void perf_pmu_migrate_context(struct pmu *pmu,
1156                                 int src_cpu, int dst_cpu);
1157 extern u64 perf_event_read_value(struct perf_event *event,
1158                                  u64 *enabled, u64 *running);
1159
1160
1161 struct perf_sample_data {
1162         u64                             type;
1163
1164         u64                             ip;
1165         struct {
1166                 u32     pid;
1167                 u32     tid;
1168         }                               tid_entry;
1169         u64                             time;
1170         u64                             addr;
1171         u64                             id;
1172         u64                             stream_id;
1173         struct {
1174                 u32     cpu;
1175                 u32     reserved;
1176         }                               cpu_entry;
1177         u64                             period;
1178         struct perf_callchain_entry     *callchain;
1179         struct perf_raw_record          *raw;
1180         struct perf_branch_stack        *br_stack;
1181         struct perf_regs_user           regs_user;
1182         u64                             stack_user_size;
1183 };
1184
1185 static inline void perf_sample_data_init(struct perf_sample_data *data,
1186                                          u64 addr, u64 period)
1187 {
1188         /* remaining struct members initialized in perf_prepare_sample() */
1189         data->addr = addr;
1190         data->raw  = NULL;
1191         data->br_stack = NULL;
1192         data->period = period;
1193         data->regs_user.abi = PERF_SAMPLE_REGS_ABI_NONE;
1194         data->regs_user.regs = NULL;
1195         data->stack_user_size = 0;
1196 }
1197
1198 extern void perf_output_sample(struct perf_output_handle *handle,
1199                                struct perf_event_header *header,
1200                                struct perf_sample_data *data,
1201                                struct perf_event *event);
1202 extern void perf_prepare_sample(struct perf_event_header *header,
1203                                 struct perf_sample_data *data,
1204                                 struct perf_event *event,
1205                                 struct pt_regs *regs);
1206
1207 extern int perf_event_overflow(struct perf_event *event,
1208                                  struct perf_sample_data *data,
1209                                  struct pt_regs *regs);
1210
1211 static inline bool is_sampling_event(struct perf_event *event)
1212 {
1213         return event->attr.sample_period != 0;
1214 }
1215
1216 /*
1217  * Return 1 for a software event, 0 for a hardware event
1218  */
1219 static inline int is_software_event(struct perf_event *event)
1220 {
1221         return event->pmu->task_ctx_nr == perf_sw_context;
1222 }
1223
1224 extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
1225
1226 extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
1227
1228 #ifndef perf_arch_fetch_caller_regs
1229 static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
1230 #endif
1231
1232 /*
1233  * Take a snapshot of the regs. Skip ip and frame pointer to
1234  * the nth caller. We only need a few of the regs:
1235  * - ip for PERF_SAMPLE_IP
1236  * - cs for user_mode() tests
1237  * - bp for callchains
1238  * - eflags, for future purposes, just in case
1239  */
1240 static inline void perf_fetch_caller_regs(struct pt_regs *regs)
1241 {
1242         memset(regs, 0, sizeof(*regs));
1243
1244         perf_arch_fetch_caller_regs(regs, CALLER_ADDR0);
1245 }
1246
1247 static __always_inline void
1248 perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
1249 {
1250         struct pt_regs hot_regs;
1251
1252         if (static_key_false(&perf_swevent_enabled[event_id])) {
1253                 if (!regs) {
1254                         perf_fetch_caller_regs(&hot_regs);
1255                         regs = &hot_regs;
1256                 }
1257                 __perf_sw_event(event_id, nr, regs, addr);
1258         }
1259 }
1260
1261 extern struct static_key_deferred perf_sched_events;
1262
1263 static inline void perf_event_task_sched_in(struct task_struct *prev,
1264                                             struct task_struct *task)
1265 {
1266         if (static_key_false(&perf_sched_events.key))
1267                 __perf_event_task_sched_in(prev, task);
1268 }
1269
1270 static inline void perf_event_task_sched_out(struct task_struct *prev,
1271                                              struct task_struct *next)
1272 {
1273         perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0);
1274
1275         if (static_key_false(&perf_sched_events.key))
1276                 __perf_event_task_sched_out(prev, next);
1277 }
1278
1279 extern void perf_event_mmap(struct vm_area_struct *vma);
1280 extern struct perf_guest_info_callbacks *perf_guest_cbs;
1281 extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
1282 extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
1283
1284 extern void perf_event_comm(struct task_struct *tsk);
1285 extern void perf_event_fork(struct task_struct *tsk);
1286
1287 /* Callchains */
1288 DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
1289
1290 extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs);
1291 extern void perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs);
1292
1293 static inline void perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
1294 {
1295         if (entry->nr < PERF_MAX_STACK_DEPTH)
1296                 entry->ip[entry->nr++] = ip;
1297 }
1298
1299 extern int sysctl_perf_event_paranoid;
1300 extern int sysctl_perf_event_mlock;
1301 extern int sysctl_perf_event_sample_rate;
1302
1303 extern int perf_proc_update_handler(struct ctl_table *table, int write,
1304                 void __user *buffer, size_t *lenp,
1305                 loff_t *ppos);
1306
1307 static inline bool perf_paranoid_tracepoint_raw(void)
1308 {
1309         return sysctl_perf_event_paranoid > -1;
1310 }
1311
1312 static inline bool perf_paranoid_cpu(void)
1313 {
1314         return sysctl_perf_event_paranoid > 0;
1315 }
1316
1317 static inline bool perf_paranoid_kernel(void)
1318 {
1319         return sysctl_perf_event_paranoid > 1;
1320 }
1321
1322 extern void perf_event_init(void);
1323 extern void perf_tp_event(u64 addr, u64 count, void *record,
1324                           int entry_size, struct pt_regs *regs,
1325                           struct hlist_head *head, int rctx,
1326                           struct task_struct *task);
1327 extern void perf_bp_event(struct perf_event *event, void *data);
1328
1329 #ifndef perf_misc_flags
1330 # define perf_misc_flags(regs) \
1331                 (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
1332 # define perf_instruction_pointer(regs) instruction_pointer(regs)
1333 #endif
1334
1335 static inline bool has_branch_stack(struct perf_event *event)
1336 {
1337         return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;
1338 }
1339
1340 extern int perf_output_begin(struct perf_output_handle *handle,
1341                              struct perf_event *event, unsigned int size);
1342 extern void perf_output_end(struct perf_output_handle *handle);
1343 extern unsigned int perf_output_copy(struct perf_output_handle *handle,
1344                              const void *buf, unsigned int len);
1345 extern unsigned int perf_output_skip(struct perf_output_handle *handle,
1346                                      unsigned int len);
1347 extern int perf_swevent_get_recursion_context(void);
1348 extern void perf_swevent_put_recursion_context(int rctx);
1349 extern void perf_event_enable(struct perf_event *event);
1350 extern void perf_event_disable(struct perf_event *event);
1351 extern int __perf_event_disable(void *info);
1352 extern void perf_event_task_tick(void);
1353 #else
1354 static inline void
1355 perf_event_task_sched_in(struct task_struct *prev,
1356                          struct task_struct *task)                      { }
1357 static inline void
1358 perf_event_task_sched_out(struct task_struct *prev,
1359                           struct task_struct *next)                     { }
1360 static inline int perf_event_init_task(struct task_struct *child)       { return 0; }
1361 static inline void perf_event_exit_task(struct task_struct *child)      { }
1362 static inline void perf_event_free_task(struct task_struct *task)       { }
1363 static inline void perf_event_delayed_put(struct task_struct *task)     { }
1364 static inline void perf_event_print_debug(void)                         { }
1365 static inline int perf_event_task_disable(void)                         { return -EINVAL; }
1366 static inline int perf_event_task_enable(void)                          { return -EINVAL; }
1367 static inline int perf_event_refresh(struct perf_event *event, int refresh)
1368 {
1369         return -EINVAL;
1370 }
1371
1372 static inline void
1373 perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)     { }
1374 static inline void
1375 perf_bp_event(struct perf_event *event, void *data)                     { }
1376
1377 static inline int perf_register_guest_info_callbacks
1378 (struct perf_guest_info_callbacks *callbacks)                           { return 0; }
1379 static inline int perf_unregister_guest_info_callbacks
1380 (struct perf_guest_info_callbacks *callbacks)                           { return 0; }
1381
1382 static inline void perf_event_mmap(struct vm_area_struct *vma)          { }
1383 static inline void perf_event_comm(struct task_struct *tsk)             { }
1384 static inline void perf_event_fork(struct task_struct *tsk)             { }
1385 static inline void perf_event_init(void)                                { }
1386 static inline int  perf_swevent_get_recursion_context(void)             { return -1; }
1387 static inline void perf_swevent_put_recursion_context(int rctx)         { }
1388 static inline void perf_event_enable(struct perf_event *event)          { }
1389 static inline void perf_event_disable(struct perf_event *event)         { }
1390 static inline int __perf_event_disable(void *info)                      { return -1; }
1391 static inline void perf_event_task_tick(void)                           { }
1392 #endif
1393
1394 #define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
1395
1396 /*
1397  * This has to have a higher priority than migration_notifier in sched.c.
1398  */
1399 #define perf_cpu_notifier(fn)                                           \
1400 do {                                                                    \
1401         static struct notifier_block fn##_nb __cpuinitdata =            \
1402                 { .notifier_call = fn, .priority = CPU_PRI_PERF };      \
1403         fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE,                     \
1404                 (void *)(unsigned long)smp_processor_id());             \
1405         fn(&fn##_nb, (unsigned long)CPU_STARTING,                       \
1406                 (void *)(unsigned long)smp_processor_id());             \
1407         fn(&fn##_nb, (unsigned long)CPU_ONLINE,                         \
1408                 (void *)(unsigned long)smp_processor_id());             \
1409         register_cpu_notifier(&fn##_nb);                                \
1410 } while (0)
1411
1412
1413 #define PMU_FORMAT_ATTR(_name, _format)                                 \
1414 static ssize_t                                                          \
1415 _name##_show(struct device *dev,                                        \
1416                                struct device_attribute *attr,           \
1417                                char *page)                              \
1418 {                                                                       \
1419         BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE);                     \
1420         return sprintf(page, _format "\n");                             \
1421 }                                                                       \
1422                                                                         \
1423 static struct device_attribute format_attr_##_name = __ATTR_RO(_name)
1424
1425 #endif /* __KERNEL__ */
1426 #endif /* _LINUX_PERF_EVENT_H */