arch/x86/events/intel/bts.c

   1 /*
   2  * BTS PMU driver for perf
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  */
  14
  15 #undef DEBUG
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/bitops.h>
  20 #include <linux/types.h>
  21 #include <linux/slab.h>
  22 #include <linux/debugfs.h>
  23 #include <linux/device.h>
  24 #include <linux/coredump.h>
  25
  26 #include <asm-generic/sizes.h>
  27 #include <asm/perf_event.h>
  28
  29 #include "../perf_event.h"
  30
  31 struct bts_ctx {
  32         struct perf_output_handle       handle;
  33         struct debug_store              ds_back;
  34         int                             started;
  35 };
  36
  37 static DEFINE_PER_CPU(struct bts_ctx, bts_ctx);
  38
  39 #define BTS_RECORD_SIZE         24
  40 #define BTS_SAFETY_MARGIN       4080
  41
  42 struct bts_phys {
  43         struct page     *page;
  44         unsigned long   size;
  45         unsigned long   offset;
  46         unsigned long   displacement;
  47 };
  48
  49 struct bts_buffer {
  50         size_t          real_size;      /* multiple of BTS_RECORD_SIZE */
  51         unsigned int    nr_pages;
  52         unsigned int    nr_bufs;
  53         unsigned int    cur_buf;
  54         bool            snapshot;
  55         local_t         data_size;
  56         local_t         lost;
  57         local_t         head;
  58         unsigned long   end;
  59         void            **data_pages;
  60         struct bts_phys buf[0];
  61 };
  62
  63 struct pmu bts_pmu;
  64
  65 static size_t buf_size(struct page *page)
  66 {
  67         return 1 << (PAGE_SHIFT + page_private(page));
  68 }
  69
  70 static void *
  71 bts_buffer_setup_aux(int cpu, void **pages, int nr_pages, bool overwrite)
  72 {
  73         struct bts_buffer *buf;
  74         struct page *page;
  75         int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
  76         unsigned long offset;
  77         size_t size = nr_pages << PAGE_SHIFT;
  78         int pg, nbuf, pad;
  79
  80         /* count all the high order buffers */
  81         for (pg = 0, nbuf = 0; pg < nr_pages;) {
  82                 page = virt_to_page(pages[pg]);
  83                 if (WARN_ON_ONCE(!PagePrivate(page) && nr_pages > 1))
  84                         return NULL;
  85                 pg += 1 << page_private(page);
  86                 nbuf++;
  87         }
  88
  89         /*
  90          * to avoid interrupts in overwrite mode, only allow one physical
  91          */
  92         if (overwrite && nbuf > 1)
  93                 return NULL;
  94
  95         buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
  96         if (!buf)
  97                 return NULL;
  98
  99         buf->nr_pages = nr_pages;
 100         buf->nr_bufs = nbuf;
 101         buf->snapshot = overwrite;
 102         buf->data_pages = pages;
 103         buf->real_size = size - size % BTS_RECORD_SIZE;
 104
 105         for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
 106                 unsigned int __nr_pages;
 107
 108                 page = virt_to_page(pages[pg]);
 109                 __nr_pages = PagePrivate(page) ? 1 << page_private(page) : 1;
 110                 buf->buf[nbuf].page = page;
 111                 buf->buf[nbuf].offset = offset;
 112                 buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
 113                 buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
 114                 pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
 115                 buf->buf[nbuf].size -= pad;
 116
 117                 pg += __nr_pages;
 118                 offset += __nr_pages << PAGE_SHIFT;
 119         }
 120
 121         return buf;
 122 }
 123
 124 static void bts_buffer_free_aux(void *data)
 125 {
 126         kfree(data);
 127 }
 128
 129 static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
 130 {
 131         return buf->buf[idx].offset + buf->buf[idx].displacement;
 132 }
 133
 134 static void
 135 bts_config_buffer(struct bts_buffer *buf)
 136 {
 137         int cpu = raw_smp_processor_id();
 138         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 139         struct bts_phys *phys = &buf->buf[buf->cur_buf];
 140         unsigned long index, thresh = 0, end = phys->size;
 141         struct page *page = phys->page;
 142
 143         index = local_read(&buf->head);
 144
 145         if (!buf->snapshot) {
 146                 if (buf->end < phys->offset + buf_size(page))
 147                         end = buf->end - phys->offset - phys->displacement;
 148
 149                 index -= phys->offset + phys->displacement;
 150
 151                 if (end - index > BTS_SAFETY_MARGIN)
 152                         thresh = end - BTS_SAFETY_MARGIN;
 153                 else if (end - index > BTS_RECORD_SIZE)
 154                         thresh = end - BTS_RECORD_SIZE;
 155                 else
 156                         thresh = end;
 157         }
 158
 159         ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;
 160         ds->bts_index = ds->bts_buffer_base + index;
 161         ds->bts_absolute_maximum = ds->bts_buffer_base + end;
 162         ds->bts_interrupt_threshold = !buf->snapshot
 163                 ? ds->bts_buffer_base + thresh
 164                 : ds->bts_absolute_maximum + BTS_RECORD_SIZE;
 165 }
 166
 167 static void bts_buffer_pad_out(struct bts_phys *phys, unsigned long head)
 168 {
 169         unsigned long index = head - phys->offset;
 170
 171         memset(page_address(phys->page) + index, 0, phys->size - index);
 172 }
 173
 174 static void bts_update(struct bts_ctx *bts)
 175 {
 176         int cpu = raw_smp_processor_id();
 177         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 178         struct bts_buffer *buf = perf_get_aux(&bts->handle);
 179         unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
 180
 181         if (!buf)
 182                 return;
 183
 184         head = index + bts_buffer_offset(buf, buf->cur_buf);
 185         old = local_xchg(&buf->head, head);
 186
 187         if (!buf->snapshot) {
 188                 if (old == head)
 189                         return;
 190
 191                 if (ds->bts_index >= ds->bts_absolute_maximum)
 192                         local_inc(&buf->lost);
 193
 194                 /*
 195                  * old and head are always in the same physical buffer, so we
 196                  * can subtract them to get the data size.
 197                  */
 198                 local_add(head - old, &buf->data_size);
 199         } else {
 200                 local_set(&buf->data_size, head);
 201         }
 202 }
 203
 204 static int
 205 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle);
 206
 207 static void __bts_event_start(struct perf_event *event)
 208 {
 209         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 210         struct bts_buffer *buf = perf_get_aux(&bts->handle);
 211         u64 config = 0;
 212
 213         if (!buf->snapshot)
 214                 config |= ARCH_PERFMON_EVENTSEL_INT;
 215         if (!event->attr.exclude_kernel)
 216                 config |= ARCH_PERFMON_EVENTSEL_OS;
 217         if (!event->attr.exclude_user)
 218                 config |= ARCH_PERFMON_EVENTSEL_USR;
 219
 220         bts_config_buffer(buf);
 221
 222         /*
 223          * local barrier to make sure that ds configuration made it
 224          * before we enable BTS
 225          */
 226         wmb();
 227
 228         intel_pmu_enable_bts(config);
 229
 230 }
 231
 232 static void bts_event_start(struct perf_event *event, int flags)
 233 {
 234         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 235         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 236         struct bts_buffer *buf;
 237
 238         buf = perf_aux_output_begin(&bts->handle, event);
 239         if (!buf)
 240                 goto fail_stop;
 241
 242         if (bts_buffer_reset(buf, &bts->handle))
 243                 goto fail_end_stop;
 244
 245         bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
 246         bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum;
 247         bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold;
 248
 249         event->hw.itrace_started = 1;
 250         event->hw.state = 0;
 251
 252         __bts_event_start(event);
 253
 254         /* PMI handler: this counter is running and likely generating PMIs */
 255         ACCESS_ONCE(bts->started) = 1;
 256
 257         return;
 258
 259 fail_end_stop:
 260         perf_aux_output_end(&bts->handle, 0, false);
 261
 262 fail_stop:
 263         event->hw.state = PERF_HES_STOPPED;
 264 }
 265
 266 static void __bts_event_stop(struct perf_event *event)
 267 {
 268         /*
 269          * No extra synchronization is mandated by the documentation to have
 270          * BTS data stores globally visible.
 271          */
 272         intel_pmu_disable_bts();
 273
 274         if (event->hw.state & PERF_HES_STOPPED)
 275                 return;
 276
 277         ACCESS_ONCE(event->hw.state) |= PERF_HES_STOPPED;
 278 }
 279
 280 static void bts_event_stop(struct perf_event *event, int flags)
 281 {
 282         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 283         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 284         struct bts_buffer *buf = perf_get_aux(&bts->handle);
 285
 286         /* PMI handler: don't restart this counter */
 287         ACCESS_ONCE(bts->started) = 0;
 288
 289         __bts_event_stop(event);
 290
 291         if (flags & PERF_EF_UPDATE) {
 292                 bts_update(bts);
 293
 294                 if (buf) {
 295                         if (buf->snapshot)
 296                                 bts->handle.head =
 297                                         local_xchg(&buf->data_size,
 298                                                    buf->nr_pages << PAGE_SHIFT);
 299                         perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),
 300                                             !!local_xchg(&buf->lost, 0));
 301                 }
 302
 303                 cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
 304                 cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base;
 305                 cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum;
 306                 cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold;
 307         }
 308 }
 309
 310 void intel_bts_enable_local(void)
 311 {
 312         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 313
 314         if (bts->handle.event && bts->started)
 315                 __bts_event_start(bts->handle.event);
 316 }
 317
 318 void intel_bts_disable_local(void)
 319 {
 320         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 321
 322         if (bts->handle.event)
 323                 __bts_event_stop(bts->handle.event);
 324 }
 325
 326 static int
 327 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
 328 {
 329         unsigned long head, space, next_space, pad, gap, skip, wakeup;
 330         unsigned int next_buf;
 331         struct bts_phys *phys, *next_phys;
 332         int ret;
 333
 334         if (buf->snapshot)
 335                 return 0;
 336
 337         head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
 338         if (WARN_ON_ONCE(head != local_read(&buf->head)))
 339                 return -EINVAL;
 340
 341         phys = &buf->buf[buf->cur_buf];
 342         space = phys->offset + phys->displacement + phys->size - head;
 343         pad = space;
 344         if (space > handle->size) {
 345                 space = handle->size;
 346                 space -= space % BTS_RECORD_SIZE;
 347         }
 348         if (space <= BTS_SAFETY_MARGIN) {
 349                 /* See if next phys buffer has more space */
 350                 next_buf = buf->cur_buf + 1;
 351                 if (next_buf >= buf->nr_bufs)
 352                         next_buf = 0;
 353                 next_phys = &buf->buf[next_buf];
 354                 gap = buf_size(phys->page) - phys->displacement - phys->size +
 355                       next_phys->displacement;
 356                 skip = pad + gap;
 357                 if (handle->size >= skip) {
 358                         next_space = next_phys->size;
 359                         if (next_space + skip > handle->size) {
 360                                 next_space = handle->size - skip;
 361                                 next_space -= next_space % BTS_RECORD_SIZE;
 362                         }
 363                         if (next_space > space || !space) {
 364                                 if (pad)
 365                                         bts_buffer_pad_out(phys, head);
 366                                 ret = perf_aux_output_skip(handle, skip);
 367                                 if (ret)
 368                                         return ret;
 369                                 /* Advance to next phys buffer */
 370                                 phys = next_phys;
 371                                 space = next_space;
 372                                 head = phys->offset + phys->displacement;
 373                                 /*
 374                                  * After this, cur_buf and head won't match ds
 375                                  * anymore, so we must not be racing with
 376                                  * bts_update().
 377                                  */
 378                                 buf->cur_buf = next_buf;
 379                                 local_set(&buf->head, head);
 380                         }
 381                 }
 382         }
 383
 384         /* Don't go far beyond wakeup watermark */
 385         wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup -
 386                  handle->head;
 387         if (space > wakeup) {
 388                 space = wakeup;
 389                 space -= space % BTS_RECORD_SIZE;
 390         }
 391
 392         buf->end = head + space;
 393
 394         /*
 395          * If we have no space, the lost notification would have been sent when
 396          * we hit absolute_maximum - see bts_update()
 397          */
 398         if (!space)
 399                 return -ENOSPC;
 400
 401         return 0;
 402 }
 403
 404 int intel_bts_interrupt(void)
 405 {
 406         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 407         struct perf_event *event = bts->handle.event;
 408         struct bts_buffer *buf;
 409         s64 old_head;
 410         int err;
 411
 412         if (!event || !bts->started)
 413                 return 0;
 414
 415         buf = perf_get_aux(&bts->handle);
 416         /*
 417          * Skip snapshot counters: they don't use the interrupt, but
 418          * there's no other way of telling, because the pointer will
 419          * keep moving
 420          */
 421         if (!buf || buf->snapshot)
 422                 return 0;
 423
 424         old_head = local_read(&buf->head);
 425         bts_update(bts);
 426
 427         /* no new data */
 428         if (old_head == local_read(&buf->head))
 429                 return 0;
 430
 431         perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),
 432                             !!local_xchg(&buf->lost, 0));
 433
 434         buf = perf_aux_output_begin(&bts->handle, event);
 435         if (!buf)
 436                 return 1;
 437
 438         err = bts_buffer_reset(buf, &bts->handle);
 439         if (err)
 440                 perf_aux_output_end(&bts->handle, 0, false);
 441
 442         return 1;
 443 }
 444
 445 static void bts_event_del(struct perf_event *event, int mode)
 446 {
 447         bts_event_stop(event, PERF_EF_UPDATE);
 448 }
 449
 450 static int bts_event_add(struct perf_event *event, int mode)
 451 {
 452         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 453         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 454         struct hw_perf_event *hwc = &event->hw;
 455
 456         event->hw.state = PERF_HES_STOPPED;
 457
 458         if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
 459                 return -EBUSY;
 460
 461         if (bts->handle.event)
 462                 return -EBUSY;
 463
 464         if (mode & PERF_EF_START) {
 465                 bts_event_start(event, 0);
 466                 if (hwc->state & PERF_HES_STOPPED)
 467                         return -EINVAL;
 468         }
 469
 470         return 0;
 471 }
 472
 473 static void bts_event_destroy(struct perf_event *event)
 474 {
 475         x86_release_hardware();
 476         x86_del_exclusive(x86_lbr_exclusive_bts);
 477 }
 478
 479 static int bts_event_init(struct perf_event *event)
 480 {
 481         int ret;
 482
 483         if (event->attr.type != bts_pmu.type)
 484                 return -ENOENT;
 485
 486         if (x86_add_exclusive(x86_lbr_exclusive_bts))
 487                 return -EBUSY;
 488
 489         /*
 490          * BTS leaks kernel addresses even when CPL0 tracing is
 491          * disabled, so disallow intel_bts driver for unprivileged
 492          * users on paranoid systems since it provides trace data
 493          * to the user in a zero-copy fashion.
 494          *
 495          * Note that the default paranoia setting permits unprivileged
 496          * users to profile the kernel.
 497          */
 498         if (event->attr.exclude_kernel && perf_paranoid_kernel() &&
 499             !capable(CAP_SYS_ADMIN))
 500                 return -EACCES;
 501
 502         ret = x86_reserve_hardware();
 503         if (ret) {
 504                 x86_del_exclusive(x86_lbr_exclusive_bts);
 505                 return ret;
 506         }
 507
 508         event->destroy = bts_event_destroy;
 509
 510         return 0;
 511 }
 512
 513 static void bts_event_read(struct perf_event *event)
 514 {
 515 }
 516
 517 static __init int bts_init(void)
 518 {
 519         if (!boot_cpu_has(X86_FEATURE_DTES64) || !x86_pmu.bts)
 520                 return -ENODEV;
 521
 522         bts_pmu.capabilities    = PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_ITRACE;
 523         bts_pmu.task_ctx_nr     = perf_sw_context;
 524         bts_pmu.event_init      = bts_event_init;
 525         bts_pmu.add             = bts_event_add;
 526         bts_pmu.del             = bts_event_del;
 527         bts_pmu.start           = bts_event_start;
 528         bts_pmu.stop            = bts_event_stop;
 529         bts_pmu.read            = bts_event_read;
 530         bts_pmu.setup_aux       = bts_buffer_setup_aux;
 531         bts_pmu.free_aux        = bts_buffer_free_aux;
 532
 533         return perf_pmu_register(&bts_pmu, "intel_bts", -1);
 534 }
 535 arch_initcall(bts_init);