kernel/perf_counter.c

   1 /*
   2  * Performance counter core code
   3  *
   4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
   7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
   8  *
   9  *  For licensing details see kernel-base/COPYING
  10  */
  11
  12 #include <linux/fs.h>
  13 #include <linux/mm.h>
  14 #include <linux/cpu.h>
  15 #include <linux/smp.h>
  16 #include <linux/file.h>
  17 #include <linux/poll.h>
  18 #include <linux/sysfs.h>
  19 #include <linux/ptrace.h>
  20 #include <linux/percpu.h>
  21 #include <linux/vmstat.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/rculist.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/syscalls.h>
  26 #include <linux/anon_inodes.h>
  27 #include <linux/kernel_stat.h>
  28 #include <linux/perf_counter.h>
  29 #include <linux/dcache.h>
  30
  31 #include <asm/irq_regs.h>
  32
  33 /*
  34  * Each CPU has a list of per CPU counters:
  35  */
  36 DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
  37
  38 int perf_max_counters __read_mostly = 1;
  39 static int perf_reserved_percpu __read_mostly;
  40 static int perf_overcommit __read_mostly = 1;
  41
  42 static atomic_t nr_counters __read_mostly;
  43 static atomic_t nr_mmap_tracking __read_mostly;
  44 static atomic_t nr_munmap_tracking __read_mostly;
  45 static atomic_t nr_comm_tracking __read_mostly;
  46
  47 int sysctl_perf_counter_priv __read_mostly; /* do we need to be privileged */
  48 int sysctl_perf_counter_mlock __read_mostly = 128; /* 'free' kb per counter */
  49
  50 /*
  51  * Lock for (sysadmin-configurable) counter reservations:
  52  */
  53 static DEFINE_SPINLOCK(perf_resource_lock);
  54
  55 /*
  56  * Architecture provided APIs - weak aliases:
  57  */
  58 extern __weak const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
  59 {
  60         return NULL;
  61 }
  62
  63 void __weak hw_perf_disable(void)               { barrier(); }
  64 void __weak hw_perf_enable(void)                { barrier(); }
  65
  66 void __weak hw_perf_counter_setup(int cpu)      { barrier(); }
  67 int __weak hw_perf_group_sched_in(struct perf_counter *group_leader,
  68                struct perf_cpu_context *cpuctx,
  69                struct perf_counter_context *ctx, int cpu)
  70 {
  71         return 0;
  72 }
  73
  74 void __weak perf_counter_print_debug(void)      { }
  75
  76 static DEFINE_PER_CPU(int, disable_count);
  77
  78 void __perf_disable(void)
  79 {
  80         __get_cpu_var(disable_count)++;
  81 }
  82
  83 bool __perf_enable(void)
  84 {
  85         return !--__get_cpu_var(disable_count);
  86 }
  87
  88 void perf_disable(void)
  89 {
  90         __perf_disable();
  91         hw_perf_disable();
  92 }
  93
  94 void perf_enable(void)
  95 {
  96         if (__perf_enable())
  97                 hw_perf_enable();
  98 }
  99
 100 static void
 101 list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
 102 {
 103         struct perf_counter *group_leader = counter->group_leader;
 104
 105         /*
 106          * Depending on whether it is a standalone or sibling counter,
 107          * add it straight to the context's counter list, or to the group
 108          * leader's sibling list:
 109          */
 110         if (group_leader == counter)
 111                 list_add_tail(&counter->list_entry, &ctx->counter_list);
 112         else {
 113                 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
 114                 group_leader->nr_siblings++;
 115         }
 116
 117         list_add_rcu(&counter->event_entry, &ctx->event_list);
 118 }
 119
 120 static void
 121 list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
 122 {
 123         struct perf_counter *sibling, *tmp;
 124
 125         list_del_init(&counter->list_entry);
 126         list_del_rcu(&counter->event_entry);
 127
 128         if (counter->group_leader != counter)
 129                 counter->group_leader->nr_siblings--;
 130
 131         /*
 132          * If this was a group counter with sibling counters then
 133          * upgrade the siblings to singleton counters by adding them
 134          * to the context list directly:
 135          */
 136         list_for_each_entry_safe(sibling, tmp,
 137                                  &counter->sibling_list, list_entry) {
 138
 139                 list_move_tail(&sibling->list_entry, &ctx->counter_list);
 140                 sibling->group_leader = sibling;
 141         }
 142 }
 143
 144 static void
 145 counter_sched_out(struct perf_counter *counter,
 146                   struct perf_cpu_context *cpuctx,
 147                   struct perf_counter_context *ctx)
 148 {
 149         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
 150                 return;
 151
 152         counter->state = PERF_COUNTER_STATE_INACTIVE;
 153         counter->tstamp_stopped = ctx->time;
 154         counter->pmu->disable(counter);
 155         counter->oncpu = -1;
 156
 157         if (!is_software_counter(counter))
 158                 cpuctx->active_oncpu--;
 159         ctx->nr_active--;
 160         if (counter->hw_event.exclusive || !cpuctx->active_oncpu)
 161                 cpuctx->exclusive = 0;
 162 }
 163
 164 static void
 165 group_sched_out(struct perf_counter *group_counter,
 166                 struct perf_cpu_context *cpuctx,
 167                 struct perf_counter_context *ctx)
 168 {
 169         struct perf_counter *counter;
 170
 171         if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
 172                 return;
 173
 174         counter_sched_out(group_counter, cpuctx, ctx);
 175
 176         /*
 177          * Schedule out siblings (if any):
 178          */
 179         list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
 180                 counter_sched_out(counter, cpuctx, ctx);
 181
 182         if (group_counter->hw_event.exclusive)
 183                 cpuctx->exclusive = 0;
 184 }
 185
 186 /*
 187  * Cross CPU call to remove a performance counter
 188  *
 189  * We disable the counter on the hardware level first. After that we
 190  * remove it from the context list.
 191  */
 192 static void __perf_counter_remove_from_context(void *info)
 193 {
 194         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 195         struct perf_counter *counter = info;
 196         struct perf_counter_context *ctx = counter->ctx;
 197         unsigned long flags;
 198
 199         /*
 200          * If this is a task context, we need to check whether it is
 201          * the current task context of this cpu. If not it has been
 202          * scheduled out before the smp call arrived.
 203          */
 204         if (ctx->task && cpuctx->task_ctx != ctx)
 205                 return;
 206
 207         spin_lock_irqsave(&ctx->lock, flags);
 208
 209         counter_sched_out(counter, cpuctx, ctx);
 210
 211         counter->task = NULL;
 212         ctx->nr_counters--;
 213
 214         /*
 215          * Protect the list operation against NMI by disabling the
 216          * counters on a global level. NOP for non NMI based counters.
 217          */
 218         perf_disable();
 219         list_del_counter(counter, ctx);
 220         perf_enable();
 221
 222         if (!ctx->task) {
 223                 /*
 224                  * Allow more per task counters with respect to the
 225                  * reservation:
 226                  */
 227                 cpuctx->max_pertask =
 228                         min(perf_max_counters - ctx->nr_counters,
 229                             perf_max_counters - perf_reserved_percpu);
 230         }
 231
 232         spin_unlock_irqrestore(&ctx->lock, flags);
 233 }
 234
 235
 236 /*
 237  * Remove the counter from a task's (or a CPU's) list of counters.
 238  *
 239  * Must be called with counter->mutex and ctx->mutex held.
 240  *
 241  * CPU counters are removed with a smp call. For task counters we only
 242  * call when the task is on a CPU.
 243  */
 244 static void perf_counter_remove_from_context(struct perf_counter *counter)
 245 {
 246         struct perf_counter_context *ctx = counter->ctx;
 247         struct task_struct *task = ctx->task;
 248
 249         if (!task) {
 250                 /*
 251                  * Per cpu counters are removed via an smp call and
 252                  * the removal is always sucessful.
 253                  */
 254                 smp_call_function_single(counter->cpu,
 255                                          __perf_counter_remove_from_context,
 256                                          counter, 1);
 257                 return;
 258         }
 259
 260 retry:
 261         task_oncpu_function_call(task, __perf_counter_remove_from_context,
 262                                  counter);
 263
 264         spin_lock_irq(&ctx->lock);
 265         /*
 266          * If the context is active we need to retry the smp call.
 267          */
 268         if (ctx->nr_active && !list_empty(&counter->list_entry)) {
 269                 spin_unlock_irq(&ctx->lock);
 270                 goto retry;
 271         }
 272
 273         /*
 274          * The lock prevents that this context is scheduled in so we
 275          * can remove the counter safely, if the call above did not
 276          * succeed.
 277          */
 278         if (!list_empty(&counter->list_entry)) {
 279                 ctx->nr_counters--;
 280                 list_del_counter(counter, ctx);
 281                 counter->task = NULL;
 282         }
 283         spin_unlock_irq(&ctx->lock);
 284 }
 285
 286 static inline u64 perf_clock(void)
 287 {
 288         return cpu_clock(smp_processor_id());
 289 }
 290
 291 /*
 292  * Update the record of the current time in a context.
 293  */
 294 static void update_context_time(struct perf_counter_context *ctx)
 295 {
 296         u64 now = perf_clock();
 297
 298         ctx->time += now - ctx->timestamp;
 299         ctx->timestamp = now;
 300 }
 301
 302 /*
 303  * Update the total_time_enabled and total_time_running fields for a counter.
 304  */
 305 static void update_counter_times(struct perf_counter *counter)
 306 {
 307         struct perf_counter_context *ctx = counter->ctx;
 308         u64 run_end;
 309
 310         if (counter->state < PERF_COUNTER_STATE_INACTIVE)
 311                 return;
 312
 313         counter->total_time_enabled = ctx->time - counter->tstamp_enabled;
 314
 315         if (counter->state == PERF_COUNTER_STATE_INACTIVE)
 316                 run_end = counter->tstamp_stopped;
 317         else
 318                 run_end = ctx->time;
 319
 320         counter->total_time_running = run_end - counter->tstamp_running;
 321 }
 322
 323 /*
 324  * Update total_time_enabled and total_time_running for all counters in a group.
 325  */
 326 static void update_group_times(struct perf_counter *leader)
 327 {
 328         struct perf_counter *counter;
 329
 330         update_counter_times(leader);
 331         list_for_each_entry(counter, &leader->sibling_list, list_entry)
 332                 update_counter_times(counter);
 333 }
 334
 335 /*
 336  * Cross CPU call to disable a performance counter
 337  */
 338 static void __perf_counter_disable(void *info)
 339 {
 340         struct perf_counter *counter = info;
 341         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 342         struct perf_counter_context *ctx = counter->ctx;
 343         unsigned long flags;
 344
 345         /*
 346          * If this is a per-task counter, need to check whether this
 347          * counter's task is the current task on this cpu.
 348          */
 349         if (ctx->task && cpuctx->task_ctx != ctx)
 350                 return;
 351
 352         spin_lock_irqsave(&ctx->lock, flags);
 353
 354         /*
 355          * If the counter is on, turn it off.
 356          * If it is in error state, leave it in error state.
 357          */
 358         if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
 359                 update_context_time(ctx);
 360                 update_counter_times(counter);
 361                 if (counter == counter->group_leader)
 362                         group_sched_out(counter, cpuctx, ctx);
 363                 else
 364                         counter_sched_out(counter, cpuctx, ctx);
 365                 counter->state = PERF_COUNTER_STATE_OFF;
 366         }
 367
 368         spin_unlock_irqrestore(&ctx->lock, flags);
 369 }
 370
 371 /*
 372  * Disable a counter.
 373  */
 374 static void perf_counter_disable(struct perf_counter *counter)
 375 {
 376         struct perf_counter_context *ctx = counter->ctx;
 377         struct task_struct *task = ctx->task;
 378
 379         if (!task) {
 380                 /*
 381                  * Disable the counter on the cpu that it's on
 382                  */
 383                 smp_call_function_single(counter->cpu, __perf_counter_disable,
 384                                          counter, 1);
 385                 return;
 386         }
 387
 388  retry:
 389         task_oncpu_function_call(task, __perf_counter_disable, counter);
 390
 391         spin_lock_irq(&ctx->lock);
 392         /*
 393          * If the counter is still active, we need to retry the cross-call.
 394          */
 395         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
 396                 spin_unlock_irq(&ctx->lock);
 397                 goto retry;
 398         }
 399
 400         /*
 401          * Since we have the lock this context can't be scheduled
 402          * in, so we can change the state safely.
 403          */
 404         if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
 405                 update_counter_times(counter);
 406                 counter->state = PERF_COUNTER_STATE_OFF;
 407         }
 408
 409         spin_unlock_irq(&ctx->lock);
 410 }
 411
 412 static int
 413 counter_sched_in(struct perf_counter *counter,
 414                  struct perf_cpu_context *cpuctx,
 415                  struct perf_counter_context *ctx,
 416                  int cpu)
 417 {
 418         if (counter->state <= PERF_COUNTER_STATE_OFF)
 419                 return 0;
 420
 421         counter->state = PERF_COUNTER_STATE_ACTIVE;
 422         counter->oncpu = cpu;   /* TODO: put 'cpu' into cpuctx->cpu */
 423         /*
 424          * The new state must be visible before we turn it on in the hardware:
 425          */
 426         smp_wmb();
 427
 428         if (counter->pmu->enable(counter)) {
 429                 counter->state = PERF_COUNTER_STATE_INACTIVE;
 430                 counter->oncpu = -1;
 431                 return -EAGAIN;
 432         }
 433
 434         counter->tstamp_running += ctx->time - counter->tstamp_stopped;
 435
 436         if (!is_software_counter(counter))
 437                 cpuctx->active_oncpu++;
 438         ctx->nr_active++;
 439
 440         if (counter->hw_event.exclusive)
 441                 cpuctx->exclusive = 1;
 442
 443         return 0;
 444 }
 445
 446 static int
 447 group_sched_in(struct perf_counter *group_counter,
 448                struct perf_cpu_context *cpuctx,
 449                struct perf_counter_context *ctx,
 450                int cpu)
 451 {
 452         struct perf_counter *counter, *partial_group;
 453         int ret;
 454
 455         if (group_counter->state == PERF_COUNTER_STATE_OFF)
 456                 return 0;
 457
 458         ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
 459         if (ret)
 460                 return ret < 0 ? ret : 0;
 461
 462         group_counter->prev_state = group_counter->state;
 463         if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
 464                 return -EAGAIN;
 465
 466         /*
 467          * Schedule in siblings as one group (if any):
 468          */
 469         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
 470                 counter->prev_state = counter->state;
 471                 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
 472                         partial_group = counter;
 473                         goto group_error;
 474                 }
 475         }
 476
 477         return 0;
 478
 479 group_error:
 480         /*
 481          * Groups can be scheduled in as one unit only, so undo any
 482          * partial group before returning:
 483          */
 484         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
 485                 if (counter == partial_group)
 486                         break;
 487                 counter_sched_out(counter, cpuctx, ctx);
 488         }
 489         counter_sched_out(group_counter, cpuctx, ctx);
 490
 491         return -EAGAIN;
 492 }
 493
 494 /*
 495  * Return 1 for a group consisting entirely of software counters,
 496  * 0 if the group contains any hardware counters.
 497  */
 498 static int is_software_only_group(struct perf_counter *leader)
 499 {
 500         struct perf_counter *counter;
 501
 502         if (!is_software_counter(leader))
 503                 return 0;
 504
 505         list_for_each_entry(counter, &leader->sibling_list, list_entry)
 506                 if (!is_software_counter(counter))
 507                         return 0;
 508
 509         return 1;
 510 }
 511
 512 /*
 513  * Work out whether we can put this counter group on the CPU now.
 514  */
 515 static int group_can_go_on(struct perf_counter *counter,
 516                            struct perf_cpu_context *cpuctx,
 517                            int can_add_hw)
 518 {
 519         /*
 520          * Groups consisting entirely of software counters can always go on.
 521          */
 522         if (is_software_only_group(counter))
 523                 return 1;
 524         /*
 525          * If an exclusive group is already on, no other hardware
 526          * counters can go on.
 527          */
 528         if (cpuctx->exclusive)
 529                 return 0;
 530         /*
 531          * If this group is exclusive and there are already
 532          * counters on the CPU, it can't go on.
 533          */
 534         if (counter->hw_event.exclusive && cpuctx->active_oncpu)
 535                 return 0;
 536         /*
 537          * Otherwise, try to add it if all previous groups were able
 538          * to go on.
 539          */
 540         return can_add_hw;
 541 }
 542
 543 static void add_counter_to_ctx(struct perf_counter *counter,
 544                                struct perf_counter_context *ctx)
 545 {
 546         list_add_counter(counter, ctx);
 547         ctx->nr_counters++;
 548         counter->prev_state = PERF_COUNTER_STATE_OFF;
 549         counter->tstamp_enabled = ctx->time;
 550         counter->tstamp_running = ctx->time;
 551         counter->tstamp_stopped = ctx->time;
 552 }
 553
 554 /*
 555  * Cross CPU call to install and enable a performance counter
 556  */
 557 static void __perf_install_in_context(void *info)
 558 {
 559         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 560         struct perf_counter *counter = info;
 561         struct perf_counter_context *ctx = counter->ctx;
 562         struct perf_counter *leader = counter->group_leader;
 563         int cpu = smp_processor_id();
 564         unsigned long flags;
 565         int err;
 566
 567         /*
 568          * If this is a task context, we need to check whether it is
 569          * the current task context of this cpu. If not it has been
 570          * scheduled out before the smp call arrived.
 571          */
 572         if (ctx->task && cpuctx->task_ctx != ctx)
 573                 return;
 574
 575         spin_lock_irqsave(&ctx->lock, flags);
 576         update_context_time(ctx);
 577
 578         /*
 579          * Protect the list operation against NMI by disabling the
 580          * counters on a global level. NOP for non NMI based counters.
 581          */
 582         perf_disable();
 583
 584         add_counter_to_ctx(counter, ctx);
 585
 586         /*
 587          * Don't put the counter on if it is disabled or if
 588          * it is in a group and the group isn't on.
 589          */
 590         if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
 591             (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
 592                 goto unlock;
 593
 594         /*
 595          * An exclusive counter can't go on if there are already active
 596          * hardware counters, and no hardware counter can go on if there
 597          * is already an exclusive counter on.
 598          */
 599         if (!group_can_go_on(counter, cpuctx, 1))
 600                 err = -EEXIST;
 601         else
 602                 err = counter_sched_in(counter, cpuctx, ctx, cpu);
 603
 604         if (err) {
 605                 /*
 606                  * This counter couldn't go on.  If it is in a group
 607                  * then we have to pull the whole group off.
 608                  * If the counter group is pinned then put it in error state.
 609                  */
 610                 if (leader != counter)
 611                         group_sched_out(leader, cpuctx, ctx);
 612                 if (leader->hw_event.pinned) {
 613                         update_group_times(leader);
 614                         leader->state = PERF_COUNTER_STATE_ERROR;
 615                 }
 616         }
 617
 618         if (!err && !ctx->task && cpuctx->max_pertask)
 619                 cpuctx->max_pertask--;
 620
 621  unlock:
 622         perf_enable();
 623
 624         spin_unlock_irqrestore(&ctx->lock, flags);
 625 }
 626
 627 /*
 628  * Attach a performance counter to a context
 629  *
 630  * First we add the counter to the list with the hardware enable bit
 631  * in counter->hw_config cleared.
 632  *
 633  * If the counter is attached to a task which is on a CPU we use a smp
 634  * call to enable it in the task context. The task might have been
 635  * scheduled away, but we check this in the smp call again.
 636  *
 637  * Must be called with ctx->mutex held.
 638  */
 639 static void
 640 perf_install_in_context(struct perf_counter_context *ctx,
 641                         struct perf_counter *counter,
 642                         int cpu)
 643 {
 644         struct task_struct *task = ctx->task;
 645
 646         if (!task) {
 647                 /*
 648                  * Per cpu counters are installed via an smp call and
 649                  * the install is always sucessful.
 650                  */
 651                 smp_call_function_single(cpu, __perf_install_in_context,
 652                                          counter, 1);
 653                 return;
 654         }
 655
 656         counter->task = task;
 657 retry:
 658         task_oncpu_function_call(task, __perf_install_in_context,
 659                                  counter);
 660
 661         spin_lock_irq(&ctx->lock);
 662         /*
 663          * we need to retry the smp call.
 664          */
 665         if (ctx->is_active && list_empty(&counter->list_entry)) {
 666                 spin_unlock_irq(&ctx->lock);
 667                 goto retry;
 668         }
 669
 670         /*
 671          * The lock prevents that this context is scheduled in so we
 672          * can add the counter safely, if it the call above did not
 673          * succeed.
 674          */
 675         if (list_empty(&counter->list_entry))
 676                 add_counter_to_ctx(counter, ctx);
 677         spin_unlock_irq(&ctx->lock);
 678 }
 679
 680 /*
 681  * Cross CPU call to enable a performance counter
 682  */
 683 static void __perf_counter_enable(void *info)
 684 {
 685         struct perf_counter *counter = info;
 686         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 687         struct perf_counter_context *ctx = counter->ctx;
 688         struct perf_counter *leader = counter->group_leader;
 689         unsigned long flags;
 690         int err;
 691
 692         /*
 693          * If this is a per-task counter, need to check whether this
 694          * counter's task is the current task on this cpu.
 695          */
 696         if (ctx->task && cpuctx->task_ctx != ctx)
 697                 return;
 698
 699         spin_lock_irqsave(&ctx->lock, flags);
 700         update_context_time(ctx);
 701
 702         counter->prev_state = counter->state;
 703         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
 704                 goto unlock;
 705         counter->state = PERF_COUNTER_STATE_INACTIVE;
 706         counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
 707
 708         /*
 709          * If the counter is in a group and isn't the group leader,
 710          * then don't put it on unless the group is on.
 711          */
 712         if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
 713                 goto unlock;
 714
 715         if (!group_can_go_on(counter, cpuctx, 1)) {
 716                 err = -EEXIST;
 717         } else {
 718                 perf_disable();
 719                 if (counter == leader)
 720                         err = group_sched_in(counter, cpuctx, ctx,
 721                                              smp_processor_id());
 722                 else
 723                         err = counter_sched_in(counter, cpuctx, ctx,
 724                                                smp_processor_id());
 725                 perf_enable();
 726         }
 727
 728         if (err) {
 729                 /*
 730                  * If this counter can't go on and it's part of a
 731                  * group, then the whole group has to come off.
 732                  */
 733                 if (leader != counter)
 734                         group_sched_out(leader, cpuctx, ctx);
 735                 if (leader->hw_event.pinned) {
 736                         update_group_times(leader);
 737                         leader->state = PERF_COUNTER_STATE_ERROR;
 738                 }
 739         }
 740
 741  unlock:
 742         spin_unlock_irqrestore(&ctx->lock, flags);
 743 }
 744
 745 /*
 746  * Enable a counter.
 747  */
 748 static void perf_counter_enable(struct perf_counter *counter)
 749 {
 750         struct perf_counter_context *ctx = counter->ctx;
 751         struct task_struct *task = ctx->task;
 752
 753         if (!task) {
 754                 /*
 755                  * Enable the counter on the cpu that it's on
 756                  */
 757                 smp_call_function_single(counter->cpu, __perf_counter_enable,
 758                                          counter, 1);
 759                 return;
 760         }
 761
 762         spin_lock_irq(&ctx->lock);
 763         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
 764                 goto out;
 765
 766         /*
 767          * If the counter is in error state, clear that first.
 768          * That way, if we see the counter in error state below, we
 769          * know that it has gone back into error state, as distinct
 770          * from the task having been scheduled away before the
 771          * cross-call arrived.
 772          */
 773         if (counter->state == PERF_COUNTER_STATE_ERROR)
 774                 counter->state = PERF_COUNTER_STATE_OFF;
 775
 776  retry:
 777         spin_unlock_irq(&ctx->lock);
 778         task_oncpu_function_call(task, __perf_counter_enable, counter);
 779
 780         spin_lock_irq(&ctx->lock);
 781
 782         /*
 783          * If the context is active and the counter is still off,
 784          * we need to retry the cross-call.
 785          */
 786         if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
 787                 goto retry;
 788
 789         /*
 790          * Since we have the lock this context can't be scheduled
 791          * in, so we can change the state safely.
 792          */
 793         if (counter->state == PERF_COUNTER_STATE_OFF) {
 794                 counter->state = PERF_COUNTER_STATE_INACTIVE;
 795                 counter->tstamp_enabled =
 796                         ctx->time - counter->total_time_enabled;
 797         }
 798  out:
 799         spin_unlock_irq(&ctx->lock);
 800 }
 801
 802 static int perf_counter_refresh(struct perf_counter *counter, int refresh)
 803 {
 804         /*
 805          * not supported on inherited counters
 806          */
 807         if (counter->hw_event.inherit)
 808                 return -EINVAL;
 809
 810         atomic_add(refresh, &counter->event_limit);
 811         perf_counter_enable(counter);
 812
 813         return 0;
 814 }
 815
 816 void __perf_counter_sched_out(struct perf_counter_context *ctx,
 817                               struct perf_cpu_context *cpuctx)
 818 {
 819         struct perf_counter *counter;
 820
 821         spin_lock(&ctx->lock);
 822         ctx->is_active = 0;
 823         if (likely(!ctx->nr_counters))
 824                 goto out;
 825         update_context_time(ctx);
 826
 827         perf_disable();
 828         if (ctx->nr_active) {
 829                 list_for_each_entry(counter, &ctx->counter_list, list_entry)
 830                         group_sched_out(counter, cpuctx, ctx);
 831         }
 832         perf_enable();
 833  out:
 834         spin_unlock(&ctx->lock);
 835 }
 836
 837 /*
 838  * Called from scheduler to remove the counters of the current task,
 839  * with interrupts disabled.
 840  *
 841  * We stop each counter and update the counter value in counter->count.
 842  *
 843  * This does not protect us against NMI, but disable()
 844  * sets the disabled bit in the control field of counter _before_
 845  * accessing the counter control register. If a NMI hits, then it will
 846  * not restart the counter.
 847  */
 848 void perf_counter_task_sched_out(struct task_struct *task, int cpu)
 849 {
 850         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
 851         struct perf_counter_context *ctx = &task->perf_counter_ctx;
 852         struct pt_regs *regs;
 853
 854         if (likely(!cpuctx->task_ctx))
 855                 return;
 856
 857         update_context_time(ctx);
 858
 859         regs = task_pt_regs(task);
 860         perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs, 0);
 861         __perf_counter_sched_out(ctx, cpuctx);
 862
 863         cpuctx->task_ctx = NULL;
 864 }
 865
 866 static void __perf_counter_task_sched_out(struct perf_counter_context *ctx)
 867 {
 868         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 869
 870         __perf_counter_sched_out(ctx, cpuctx);
 871         cpuctx->task_ctx = NULL;
 872 }
 873
 874 static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
 875 {
 876         __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
 877 }
 878
 879 static void
 880 __perf_counter_sched_in(struct perf_counter_context *ctx,
 881                         struct perf_cpu_context *cpuctx, int cpu)
 882 {
 883         struct perf_counter *counter;
 884         int can_add_hw = 1;
 885
 886         spin_lock(&ctx->lock);
 887         ctx->is_active = 1;
 888         if (likely(!ctx->nr_counters))
 889                 goto out;
 890
 891         ctx->timestamp = perf_clock();
 892
 893         perf_disable();
 894
 895         /*
 896          * First go through the list and put on any pinned groups
 897          * in order to give them the best chance of going on.
 898          */
 899         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 900                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
 901                     !counter->hw_event.pinned)
 902                         continue;
 903                 if (counter->cpu != -1 && counter->cpu != cpu)
 904                         continue;
 905
 906                 if (group_can_go_on(counter, cpuctx, 1))
 907                         group_sched_in(counter, cpuctx, ctx, cpu);
 908
 909                 /*
 910                  * If this pinned group hasn't been scheduled,
 911                  * put it in error state.
 912                  */
 913                 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
 914                         update_group_times(counter);
 915                         counter->state = PERF_COUNTER_STATE_ERROR;
 916                 }
 917         }
 918
 919         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 920                 /*
 921                  * Ignore counters in OFF or ERROR state, and
 922                  * ignore pinned counters since we did them already.
 923                  */
 924                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
 925                     counter->hw_event.pinned)
 926                         continue;
 927
 928                 /*
 929                  * Listen to the 'cpu' scheduling filter constraint
 930                  * of counters:
 931                  */
 932                 if (counter->cpu != -1 && counter->cpu != cpu)
 933                         continue;
 934
 935                 if (group_can_go_on(counter, cpuctx, can_add_hw)) {
 936                         if (group_sched_in(counter, cpuctx, ctx, cpu))
 937                                 can_add_hw = 0;
 938                 }
 939         }
 940         perf_enable();
 941  out:
 942         spin_unlock(&ctx->lock);
 943 }
 944
 945 /*
 946  * Called from scheduler to add the counters of the current task
 947  * with interrupts disabled.
 948  *
 949  * We restore the counter value and then enable it.
 950  *
 951  * This does not protect us against NMI, but enable()
 952  * sets the enabled bit in the control field of counter _before_
 953  * accessing the counter control register. If a NMI hits, then it will
 954  * keep the counter running.
 955  */
 956 void perf_counter_task_sched_in(struct task_struct *task, int cpu)
 957 {
 958         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
 959         struct perf_counter_context *ctx = &task->perf_counter_ctx;
 960
 961         __perf_counter_sched_in(ctx, cpuctx, cpu);
 962         cpuctx->task_ctx = ctx;
 963 }
 964
 965 static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
 966 {
 967         struct perf_counter_context *ctx = &cpuctx->ctx;
 968
 969         __perf_counter_sched_in(ctx, cpuctx, cpu);
 970 }
 971
 972 int perf_counter_task_disable(void)
 973 {
 974         struct task_struct *curr = current;
 975         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
 976         struct perf_counter *counter;
 977         unsigned long flags;
 978
 979         if (likely(!ctx->nr_counters))
 980                 return 0;
 981
 982         local_irq_save(flags);
 983
 984         __perf_counter_task_sched_out(ctx);
 985
 986         spin_lock(&ctx->lock);
 987
 988         /*
 989          * Disable all the counters:
 990          */
 991         perf_disable();
 992
 993         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 994                 if (counter->state != PERF_COUNTER_STATE_ERROR) {
 995                         update_group_times(counter);
 996                         counter->state = PERF_COUNTER_STATE_OFF;
 997                 }
 998         }
 999
1000         perf_enable();
1001
1002         spin_unlock_irqrestore(&ctx->lock, flags);
1003
1004         return 0;
1005 }
1006
1007 int perf_counter_task_enable(void)
1008 {
1009         struct task_struct *curr = current;
1010         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
1011         struct perf_counter *counter;
1012         unsigned long flags;
1013         int cpu;
1014
1015         if (likely(!ctx->nr_counters))
1016                 return 0;
1017
1018         local_irq_save(flags);
1019         cpu = smp_processor_id();
1020
1021         __perf_counter_task_sched_out(ctx);
1022
1023         spin_lock(&ctx->lock);
1024
1025         /*
1026          * Disable all the counters:
1027          */
1028         perf_disable();
1029
1030         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1031                 if (counter->state > PERF_COUNTER_STATE_OFF)
1032                         continue;
1033                 counter->state = PERF_COUNTER_STATE_INACTIVE;
1034                 counter->tstamp_enabled =
1035                         ctx->time - counter->total_time_enabled;
1036                 counter->hw_event.disabled = 0;
1037         }
1038         perf_enable();
1039
1040         spin_unlock(&ctx->lock);
1041
1042         perf_counter_task_sched_in(curr, cpu);
1043
1044         local_irq_restore(flags);
1045
1046         return 0;
1047 }
1048
1049 /*
1050  * Round-robin a context's counters:
1051  */
1052 static void rotate_ctx(struct perf_counter_context *ctx)
1053 {
1054         struct perf_counter *counter;
1055
1056         if (!ctx->nr_counters)
1057                 return;
1058
1059         spin_lock(&ctx->lock);
1060         /*
1061          * Rotate the first entry last (works just fine for group counters too):
1062          */
1063         perf_disable();
1064         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1065                 list_move_tail(&counter->list_entry, &ctx->counter_list);
1066                 break;
1067         }
1068         perf_enable();
1069
1070         spin_unlock(&ctx->lock);
1071 }
1072
1073 void perf_counter_task_tick(struct task_struct *curr, int cpu)
1074 {
1075         struct perf_cpu_context *cpuctx;
1076         struct perf_counter_context *ctx;
1077
1078         if (!atomic_read(&nr_counters))
1079                 return;
1080
1081         cpuctx = &per_cpu(perf_cpu_context, cpu);
1082         ctx = &curr->perf_counter_ctx;
1083
1084         perf_counter_cpu_sched_out(cpuctx);
1085         __perf_counter_task_sched_out(ctx);
1086
1087         rotate_ctx(&cpuctx->ctx);
1088         rotate_ctx(ctx);
1089
1090         perf_counter_cpu_sched_in(cpuctx, cpu);
1091         perf_counter_task_sched_in(curr, cpu);
1092 }
1093
1094 /*
1095  * Cross CPU call to read the hardware counter
1096  */
1097 static void __read(void *info)
1098 {
1099         struct perf_counter *counter = info;
1100         struct perf_counter_context *ctx = counter->ctx;
1101         unsigned long flags;
1102
1103         local_irq_save(flags);
1104         if (ctx->is_active)
1105                 update_context_time(ctx);
1106         counter->pmu->read(counter);
1107         update_counter_times(counter);
1108         local_irq_restore(flags);
1109 }
1110
1111 static u64 perf_counter_read(struct perf_counter *counter)
1112 {
1113         /*
1114          * If counter is enabled and currently active on a CPU, update the
1115          * value in the counter structure:
1116          */
1117         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
1118                 smp_call_function_single(counter->oncpu,
1119                                          __read, counter, 1);
1120         } else if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1121                 update_counter_times(counter);
1122         }
1123
1124         return atomic64_read(&counter->count);
1125 }
1126
1127 static void put_context(struct perf_counter_context *ctx)
1128 {
1129         if (ctx->task)
1130                 put_task_struct(ctx->task);
1131 }
1132
1133 static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1134 {
1135         struct perf_cpu_context *cpuctx;
1136         struct perf_counter_context *ctx;
1137         struct task_struct *task;
1138
1139         /*
1140          * If cpu is not a wildcard then this is a percpu counter:
1141          */
1142         if (cpu != -1) {
1143                 /* Must be root to operate on a CPU counter: */
1144                 if (sysctl_perf_counter_priv && !capable(CAP_SYS_ADMIN))
1145                         return ERR_PTR(-EACCES);
1146
1147                 if (cpu < 0 || cpu > num_possible_cpus())
1148                         return ERR_PTR(-EINVAL);
1149
1150                 /*
1151                  * We could be clever and allow to attach a counter to an
1152                  * offline CPU and activate it when the CPU comes up, but
1153                  * that's for later.
1154                  */
1155                 if (!cpu_isset(cpu, cpu_online_map))
1156                         return ERR_PTR(-ENODEV);
1157
1158                 cpuctx = &per_cpu(perf_cpu_context, cpu);
1159                 ctx = &cpuctx->ctx;
1160
1161                 return ctx;
1162         }
1163
1164         rcu_read_lock();
1165         if (!pid)
1166                 task = current;
1167         else
1168                 task = find_task_by_vpid(pid);
1169         if (task)
1170                 get_task_struct(task);
1171         rcu_read_unlock();
1172
1173         if (!task)
1174                 return ERR_PTR(-ESRCH);
1175
1176         ctx = &task->perf_counter_ctx;
1177         ctx->task = task;
1178
1179         /* Reuse ptrace permission checks for now. */
1180         if (!ptrace_may_access(task, PTRACE_MODE_READ)) {
1181                 put_context(ctx);
1182                 return ERR_PTR(-EACCES);
1183         }
1184
1185         return ctx;
1186 }
1187
1188 static void free_counter_rcu(struct rcu_head *head)
1189 {
1190         struct perf_counter *counter;
1191
1192         counter = container_of(head, struct perf_counter, rcu_head);
1193         kfree(counter);
1194 }
1195
1196 static void perf_pending_sync(struct perf_counter *counter);
1197
1198 static void free_counter(struct perf_counter *counter)
1199 {
1200         perf_pending_sync(counter);
1201
1202         atomic_dec(&nr_counters);
1203         if (counter->hw_event.mmap)
1204                 atomic_dec(&nr_mmap_tracking);
1205         if (counter->hw_event.munmap)
1206                 atomic_dec(&nr_munmap_tracking);
1207         if (counter->hw_event.comm)
1208                 atomic_dec(&nr_comm_tracking);
1209
1210         if (counter->destroy)
1211                 counter->destroy(counter);
1212
1213         call_rcu(&counter->rcu_head, free_counter_rcu);
1214 }
1215
1216 /*
1217  * Called when the last reference to the file is gone.
1218  */
1219 static int perf_release(struct inode *inode, struct file *file)
1220 {
1221         struct perf_counter *counter = file->private_data;
1222         struct perf_counter_context *ctx = counter->ctx;
1223
1224         file->private_data = NULL;
1225
1226         mutex_lock(&ctx->mutex);
1227         mutex_lock(&counter->mutex);
1228
1229         perf_counter_remove_from_context(counter);
1230
1231         mutex_unlock(&counter->mutex);
1232         mutex_unlock(&ctx->mutex);
1233
1234         free_counter(counter);
1235         put_context(ctx);
1236
1237         return 0;
1238 }
1239
1240 /*
1241  * Read the performance counter - simple non blocking version for now
1242  */
1243 static ssize_t
1244 perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1245 {
1246         u64 values[3];
1247         int n;
1248
1249         /*
1250          * Return end-of-file for a read on a counter that is in
1251          * error state (i.e. because it was pinned but it couldn't be
1252          * scheduled on to the CPU at some point).
1253          */
1254         if (counter->state == PERF_COUNTER_STATE_ERROR)
1255                 return 0;
1256
1257         mutex_lock(&counter->mutex);
1258         values[0] = perf_counter_read(counter);
1259         n = 1;
1260         if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1261                 values[n++] = counter->total_time_enabled +
1262                         atomic64_read(&counter->child_total_time_enabled);
1263         if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1264                 values[n++] = counter->total_time_running +
1265                         atomic64_read(&counter->child_total_time_running);
1266         mutex_unlock(&counter->mutex);
1267
1268         if (count < n * sizeof(u64))
1269                 return -EINVAL;
1270         count = n * sizeof(u64);
1271
1272         if (copy_to_user(buf, values, count))
1273                 return -EFAULT;
1274
1275         return count;
1276 }
1277
1278 static ssize_t
1279 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1280 {
1281         struct perf_counter *counter = file->private_data;
1282
1283         return perf_read_hw(counter, buf, count);
1284 }
1285
1286 static unsigned int perf_poll(struct file *file, poll_table *wait)
1287 {
1288         struct perf_counter *counter = file->private_data;
1289         struct perf_mmap_data *data;
1290         unsigned int events = POLL_HUP;
1291
1292         rcu_read_lock();
1293         data = rcu_dereference(counter->data);
1294         if (data)
1295                 events = atomic_xchg(&data->poll, 0);
1296         rcu_read_unlock();
1297
1298         poll_wait(file, &counter->waitq, wait);
1299
1300         return events;
1301 }
1302
1303 static void perf_counter_reset(struct perf_counter *counter)
1304 {
1305         (void)perf_counter_read(counter);
1306         atomic64_set(&counter->count, 0);
1307         perf_counter_update_userpage(counter);
1308 }
1309
1310 static void perf_counter_for_each_sibling(struct perf_counter *counter,
1311                                           void (*func)(struct perf_counter *))
1312 {
1313         struct perf_counter_context *ctx = counter->ctx;
1314         struct perf_counter *sibling;
1315
1316         spin_lock_irq(&ctx->lock);
1317         counter = counter->group_leader;
1318
1319         func(counter);
1320         list_for_each_entry(sibling, &counter->sibling_list, list_entry)
1321                 func(sibling);
1322         spin_unlock_irq(&ctx->lock);
1323 }
1324
1325 static void perf_counter_for_each_child(struct perf_counter *counter,
1326                                         void (*func)(struct perf_counter *))
1327 {
1328         struct perf_counter *child;
1329
1330         mutex_lock(&counter->mutex);
1331         func(counter);
1332         list_for_each_entry(child, &counter->child_list, child_list)
1333                 func(child);
1334         mutex_unlock(&counter->mutex);
1335 }
1336
1337 static void perf_counter_for_each(struct perf_counter *counter,
1338                                   void (*func)(struct perf_counter *))
1339 {
1340         struct perf_counter *child;
1341
1342         mutex_lock(&counter->mutex);
1343         perf_counter_for_each_sibling(counter, func);
1344         list_for_each_entry(child, &counter->child_list, child_list)
1345                 perf_counter_for_each_sibling(child, func);
1346         mutex_unlock(&counter->mutex);
1347 }
1348
1349 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1350 {
1351         struct perf_counter *counter = file->private_data;
1352         void (*func)(struct perf_counter *);
1353         u32 flags = arg;
1354
1355         switch (cmd) {
1356         case PERF_COUNTER_IOC_ENABLE:
1357                 func = perf_counter_enable;
1358                 break;
1359         case PERF_COUNTER_IOC_DISABLE:
1360                 func = perf_counter_disable;
1361                 break;
1362         case PERF_COUNTER_IOC_RESET:
1363                 func = perf_counter_reset;
1364                 break;
1365
1366         case PERF_COUNTER_IOC_REFRESH:
1367                 return perf_counter_refresh(counter, arg);
1368         default:
1369                 return -ENOTTY;
1370         }
1371
1372         if (flags & PERF_IOC_FLAG_GROUP)
1373                 perf_counter_for_each(counter, func);
1374         else
1375                 perf_counter_for_each_child(counter, func);
1376
1377         return 0;
1378 }
1379
1380 /*
1381  * Callers need to ensure there can be no nesting of this function, otherwise
1382  * the seqlock logic goes bad. We can not serialize this because the arch
1383  * code calls this from NMI context.
1384  */
1385 void perf_counter_update_userpage(struct perf_counter *counter)
1386 {
1387         struct perf_mmap_data *data;
1388         struct perf_counter_mmap_page *userpg;
1389
1390         rcu_read_lock();
1391         data = rcu_dereference(counter->data);
1392         if (!data)
1393                 goto unlock;
1394
1395         userpg = data->user_page;
1396
1397         /*
1398          * Disable preemption so as to not let the corresponding user-space
1399          * spin too long if we get preempted.
1400          */
1401         preempt_disable();
1402         ++userpg->lock;
1403         barrier();
1404         userpg->index = counter->hw.idx;
1405         userpg->offset = atomic64_read(&counter->count);
1406         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1407                 userpg->offset -= atomic64_read(&counter->hw.prev_count);
1408
1409         barrier();
1410         ++userpg->lock;
1411         preempt_enable();
1412 unlock:
1413         rcu_read_unlock();
1414 }
1415
1416 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1417 {
1418         struct perf_counter *counter = vma->vm_file->private_data;
1419         struct perf_mmap_data *data;
1420         int ret = VM_FAULT_SIGBUS;
1421
1422         rcu_read_lock();
1423         data = rcu_dereference(counter->data);
1424         if (!data)
1425                 goto unlock;
1426
1427         if (vmf->pgoff == 0) {
1428                 vmf->page = virt_to_page(data->user_page);
1429         } else {
1430                 int nr = vmf->pgoff - 1;
1431
1432                 if ((unsigned)nr > data->nr_pages)
1433                         goto unlock;
1434
1435                 vmf->page = virt_to_page(data->data_pages[nr]);
1436         }
1437         get_page(vmf->page);
1438         ret = 0;
1439 unlock:
1440         rcu_read_unlock();
1441
1442         return ret;
1443 }
1444
1445 static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
1446 {
1447         struct perf_mmap_data *data;
1448         unsigned long size;
1449         int i;
1450
1451         WARN_ON(atomic_read(&counter->mmap_count));
1452
1453         size = sizeof(struct perf_mmap_data);
1454         size += nr_pages * sizeof(void *);
1455
1456         data = kzalloc(size, GFP_KERNEL);
1457         if (!data)
1458                 goto fail;
1459
1460         data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
1461         if (!data->user_page)
1462                 goto fail_user_page;
1463
1464         for (i = 0; i < nr_pages; i++) {
1465                 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
1466                 if (!data->data_pages[i])
1467                         goto fail_data_pages;
1468         }
1469
1470         data->nr_pages = nr_pages;
1471         atomic_set(&data->lock, -1);
1472
1473         rcu_assign_pointer(counter->data, data);
1474
1475         return 0;
1476
1477 fail_data_pages:
1478         for (i--; i >= 0; i--)
1479                 free_page((unsigned long)data->data_pages[i]);
1480
1481         free_page((unsigned long)data->user_page);
1482
1483 fail_user_page:
1484         kfree(data);
1485
1486 fail:
1487         return -ENOMEM;
1488 }
1489
1490 static void __perf_mmap_data_free(struct rcu_head *rcu_head)
1491 {
1492         struct perf_mmap_data *data = container_of(rcu_head,
1493                         struct perf_mmap_data, rcu_head);
1494         int i;
1495
1496         free_page((unsigned long)data->user_page);
1497         for (i = 0; i < data->nr_pages; i++)
1498                 free_page((unsigned long)data->data_pages[i]);
1499         kfree(data);
1500 }
1501
1502 static void perf_mmap_data_free(struct perf_counter *counter)
1503 {
1504         struct perf_mmap_data *data = counter->data;
1505
1506         WARN_ON(atomic_read(&counter->mmap_count));
1507
1508         rcu_assign_pointer(counter->data, NULL);
1509         call_rcu(&data->rcu_head, __perf_mmap_data_free);
1510 }
1511
1512 static void perf_mmap_open(struct vm_area_struct *vma)
1513 {
1514         struct perf_counter *counter = vma->vm_file->private_data;
1515
1516         atomic_inc(&counter->mmap_count);
1517 }
1518
1519 static void perf_mmap_close(struct vm_area_struct *vma)
1520 {
1521         struct perf_counter *counter = vma->vm_file->private_data;
1522
1523         if (atomic_dec_and_mutex_lock(&counter->mmap_count,
1524                                       &counter->mmap_mutex)) {
1525                 vma->vm_mm->locked_vm -= counter->data->nr_locked;
1526                 perf_mmap_data_free(counter);
1527                 mutex_unlock(&counter->mmap_mutex);
1528         }
1529 }
1530
1531 static struct vm_operations_struct perf_mmap_vmops = {
1532         .open  = perf_mmap_open,
1533         .close = perf_mmap_close,
1534         .fault = perf_mmap_fault,
1535 };
1536
1537 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
1538 {
1539         struct perf_counter *counter = file->private_data;
1540         unsigned long vma_size;
1541         unsigned long nr_pages;
1542         unsigned long locked, lock_limit;
1543         int ret = 0;
1544         long extra;
1545
1546         if (!(vma->vm_flags & VM_SHARED) || (vma->vm_flags & VM_WRITE))
1547                 return -EINVAL;
1548
1549         vma_size = vma->vm_end - vma->vm_start;
1550         nr_pages = (vma_size / PAGE_SIZE) - 1;
1551
1552         /*
1553          * If we have data pages ensure they're a power-of-two number, so we
1554          * can do bitmasks instead of modulo.
1555          */
1556         if (nr_pages != 0 && !is_power_of_2(nr_pages))
1557                 return -EINVAL;
1558
1559         if (vma_size != PAGE_SIZE * (1 + nr_pages))
1560                 return -EINVAL;
1561
1562         if (vma->vm_pgoff != 0)
1563                 return -EINVAL;
1564
1565         mutex_lock(&counter->mmap_mutex);
1566         if (atomic_inc_not_zero(&counter->mmap_count)) {
1567                 if (nr_pages != counter->data->nr_pages)
1568                         ret = -EINVAL;
1569                 goto unlock;
1570         }
1571
1572         extra = nr_pages /* + 1 only account the data pages */;
1573         extra -= sysctl_perf_counter_mlock >> (PAGE_SHIFT - 10);
1574         if (extra < 0)
1575                 extra = 0;
1576
1577         locked = vma->vm_mm->locked_vm + extra;
1578
1579         lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
1580         lock_limit >>= PAGE_SHIFT;
1581
1582         if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
1583                 ret = -EPERM;
1584                 goto unlock;
1585         }
1586
1587         WARN_ON(counter->data);
1588         ret = perf_mmap_data_alloc(counter, nr_pages);
1589         if (ret)
1590                 goto unlock;
1591
1592         atomic_set(&counter->mmap_count, 1);
1593         vma->vm_mm->locked_vm += extra;
1594         counter->data->nr_locked = extra;
1595 unlock:
1596         mutex_unlock(&counter->mmap_mutex);
1597
1598         vma->vm_flags &= ~VM_MAYWRITE;
1599         vma->vm_flags |= VM_RESERVED;
1600         vma->vm_ops = &perf_mmap_vmops;
1601
1602         return ret;
1603 }
1604
1605 static int perf_fasync(int fd, struct file *filp, int on)
1606 {
1607         struct perf_counter *counter = filp->private_data;
1608         struct inode *inode = filp->f_path.dentry->d_inode;
1609         int retval;
1610
1611         mutex_lock(&inode->i_mutex);
1612         retval = fasync_helper(fd, filp, on, &counter->fasync);
1613         mutex_unlock(&inode->i_mutex);
1614
1615         if (retval < 0)
1616                 return retval;
1617
1618         return 0;
1619 }
1620
1621 static const struct file_operations perf_fops = {
1622         .release                = perf_release,
1623         .read                   = perf_read,
1624         .poll                   = perf_poll,
1625         .unlocked_ioctl         = perf_ioctl,
1626         .compat_ioctl           = perf_ioctl,
1627         .mmap                   = perf_mmap,
1628         .fasync                 = perf_fasync,
1629 };
1630
1631 /*
1632  * Perf counter wakeup
1633  *
1634  * If there's data, ensure we set the poll() state and publish everything
1635  * to user-space before waking everybody up.
1636  */
1637
1638 void perf_counter_wakeup(struct perf_counter *counter)
1639 {
1640         wake_up_all(&counter->waitq);
1641
1642         if (counter->pending_kill) {
1643                 kill_fasync(&counter->fasync, SIGIO, counter->pending_kill);
1644                 counter->pending_kill = 0;
1645         }
1646 }
1647
1648 /*
1649  * Pending wakeups
1650  *
1651  * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
1652  *
1653  * The NMI bit means we cannot possibly take locks. Therefore, maintain a
1654  * single linked list and use cmpxchg() to add entries lockless.
1655  */
1656
1657 static void perf_pending_counter(struct perf_pending_entry *entry)
1658 {
1659         struct perf_counter *counter = container_of(entry,
1660                         struct perf_counter, pending);
1661
1662         if (counter->pending_disable) {
1663                 counter->pending_disable = 0;
1664                 perf_counter_disable(counter);
1665         }
1666
1667         if (counter->pending_wakeup) {
1668                 counter->pending_wakeup = 0;
1669                 perf_counter_wakeup(counter);
1670         }
1671 }
1672
1673 #define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
1674
1675 static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
1676         PENDING_TAIL,
1677 };
1678
1679 static void perf_pending_queue(struct perf_pending_entry *entry,
1680                                void (*func)(struct perf_pending_entry *))
1681 {
1682         struct perf_pending_entry **head;
1683
1684         if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
1685                 return;
1686
1687         entry->func = func;
1688
1689         head = &get_cpu_var(perf_pending_head);
1690
1691         do {
1692                 entry->next = *head;
1693         } while (cmpxchg(head, entry->next, entry) != entry->next);
1694
1695         set_perf_counter_pending();
1696
1697         put_cpu_var(perf_pending_head);
1698 }
1699
1700 static int __perf_pending_run(void)
1701 {
1702         struct perf_pending_entry *list;
1703         int nr = 0;
1704
1705         list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
1706         while (list != PENDING_TAIL) {
1707                 void (*func)(struct perf_pending_entry *);
1708                 struct perf_pending_entry *entry = list;
1709
1710                 list = list->next;
1711
1712                 func = entry->func;
1713                 entry->next = NULL;
1714                 /*
1715                  * Ensure we observe the unqueue before we issue the wakeup,
1716                  * so that we won't be waiting forever.
1717                  * -- see perf_not_pending().
1718                  */
1719                 smp_wmb();
1720
1721                 func(entry);
1722                 nr++;
1723         }
1724
1725         return nr;
1726 }
1727
1728 static inline int perf_not_pending(struct perf_counter *counter)
1729 {
1730         /*
1731          * If we flush on whatever cpu we run, there is a chance we don't
1732          * need to wait.
1733          */
1734         get_cpu();
1735         __perf_pending_run();
1736         put_cpu();
1737
1738         /*
1739          * Ensure we see the proper queue state before going to sleep
1740          * so that we do not miss the wakeup. -- see perf_pending_handle()
1741          */
1742         smp_rmb();
1743         return counter->pending.next == NULL;
1744 }
1745
1746 static void perf_pending_sync(struct perf_counter *counter)
1747 {
1748         wait_event(counter->waitq, perf_not_pending(counter));
1749 }
1750
1751 void perf_counter_do_pending(void)
1752 {
1753         __perf_pending_run();
1754 }
1755
1756 /*
1757  * Callchain support -- arch specific
1758  */
1759
1760 __weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1761 {
1762         return NULL;
1763 }
1764
1765 /*
1766  * Output
1767  */
1768
1769 struct perf_output_handle {
1770         struct perf_counter     *counter;
1771         struct perf_mmap_data   *data;
1772         unsigned int            offset;
1773         unsigned int            head;
1774         int                     nmi;
1775         int                     overflow;
1776         int                     locked;
1777         unsigned long           flags;
1778 };
1779
1780 static void perf_output_wakeup(struct perf_output_handle *handle)
1781 {
1782         atomic_set(&handle->data->poll, POLL_IN);
1783
1784         if (handle->nmi) {
1785                 handle->counter->pending_wakeup = 1;
1786                 perf_pending_queue(&handle->counter->pending,
1787                                    perf_pending_counter);
1788         } else
1789                 perf_counter_wakeup(handle->counter);
1790 }
1791
1792 /*
1793  * Curious locking construct.
1794  *
1795  * We need to ensure a later event doesn't publish a head when a former
1796  * event isn't done writing. However since we need to deal with NMIs we
1797  * cannot fully serialize things.
1798  *
1799  * What we do is serialize between CPUs so we only have to deal with NMI
1800  * nesting on a single CPU.
1801  *
1802  * We only publish the head (and generate a wakeup) when the outer-most
1803  * event completes.
1804  */
1805 static void perf_output_lock(struct perf_output_handle *handle)
1806 {
1807         struct perf_mmap_data *data = handle->data;
1808         int cpu;
1809
1810         handle->locked = 0;
1811
1812         local_irq_save(handle->flags);
1813         cpu = smp_processor_id();
1814
1815         if (in_nmi() && atomic_read(&data->lock) == cpu)
1816                 return;
1817
1818         while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
1819                 cpu_relax();
1820
1821         handle->locked = 1;
1822 }
1823
1824 static void perf_output_unlock(struct perf_output_handle *handle)
1825 {
1826         struct perf_mmap_data *data = handle->data;
1827         int head, cpu;
1828
1829         data->done_head = data->head;
1830
1831         if (!handle->locked)
1832                 goto out;
1833
1834 again:
1835         /*
1836          * The xchg implies a full barrier that ensures all writes are done
1837          * before we publish the new head, matched by a rmb() in userspace when
1838          * reading this position.
1839          */
1840         while ((head = atomic_xchg(&data->done_head, 0)))
1841                 data->user_page->data_head = head;
1842
1843         /*
1844          * NMI can happen here, which means we can miss a done_head update.
1845          */
1846
1847         cpu = atomic_xchg(&data->lock, -1);
1848         WARN_ON_ONCE(cpu != smp_processor_id());
1849
1850         /*
1851          * Therefore we have to validate we did not indeed do so.
1852          */
1853         if (unlikely(atomic_read(&data->done_head))) {
1854                 /*
1855                  * Since we had it locked, we can lock it again.
1856                  */
1857                 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
1858                         cpu_relax();
1859
1860                 goto again;
1861         }
1862
1863         if (atomic_xchg(&data->wakeup, 0))
1864                 perf_output_wakeup(handle);
1865 out:
1866         local_irq_restore(handle->flags);
1867 }
1868
1869 static int perf_output_begin(struct perf_output_handle *handle,
1870                              struct perf_counter *counter, unsigned int size,
1871                              int nmi, int overflow)
1872 {
1873         struct perf_mmap_data *data;
1874         unsigned int offset, head;
1875
1876         /*
1877          * For inherited counters we send all the output towards the parent.
1878          */
1879         if (counter->parent)
1880                 counter = counter->parent;
1881
1882         rcu_read_lock();
1883         data = rcu_dereference(counter->data);
1884         if (!data)
1885                 goto out;
1886
1887         handle->data     = data;
1888         handle->counter  = counter;
1889         handle->nmi      = nmi;
1890         handle->overflow = overflow;
1891
1892         if (!data->nr_pages)
1893                 goto fail;
1894
1895         perf_output_lock(handle);
1896
1897         do {
1898                 offset = head = atomic_read(&data->head);
1899                 head += size;
1900         } while (atomic_cmpxchg(&data->head, offset, head) != offset);
1901
1902         handle->offset  = offset;
1903         handle->head    = head;
1904
1905         if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
1906                 atomic_set(&data->wakeup, 1);
1907
1908         return 0;
1909
1910 fail:
1911         perf_output_wakeup(handle);
1912 out:
1913         rcu_read_unlock();
1914
1915         return -ENOSPC;
1916 }
1917
1918 static void perf_output_copy(struct perf_output_handle *handle,
1919                              void *buf, unsigned int len)
1920 {
1921         unsigned int pages_mask;
1922         unsigned int offset;
1923         unsigned int size;
1924         void **pages;
1925
1926         offset          = handle->offset;
1927         pages_mask      = handle->data->nr_pages - 1;
1928         pages           = handle->data->data_pages;
1929
1930         do {
1931                 unsigned int page_offset;
1932                 int nr;
1933
1934                 nr          = (offset >> PAGE_SHIFT) & pages_mask;
1935                 page_offset = offset & (PAGE_SIZE - 1);
1936                 size        = min_t(unsigned int, PAGE_SIZE - page_offset, len);
1937
1938                 memcpy(pages[nr] + page_offset, buf, size);
1939
1940                 len         -= size;
1941                 buf         += size;
1942                 offset      += size;
1943         } while (len);
1944
1945         handle->offset = offset;
1946
1947         /*
1948          * Check we didn't copy past our reservation window, taking the
1949          * possible unsigned int wrap into account.
1950          */
1951         WARN_ON_ONCE(((int)(handle->head - handle->offset)) < 0);
1952 }
1953
1954 #define perf_output_put(handle, x) \
1955         perf_output_copy((handle), &(x), sizeof(x))
1956
1957 static void perf_output_end(struct perf_output_handle *handle)
1958 {
1959         struct perf_counter *counter = handle->counter;
1960         struct perf_mmap_data *data = handle->data;
1961
1962         int wakeup_events = counter->hw_event.wakeup_events;
1963
1964         if (handle->overflow && wakeup_events) {
1965                 int events = atomic_inc_return(&data->events);
1966                 if (events >= wakeup_events) {
1967                         atomic_sub(wakeup_events, &data->events);
1968                         atomic_set(&data->wakeup, 1);
1969                 }
1970         }
1971
1972         perf_output_unlock(handle);
1973         rcu_read_unlock();
1974 }
1975
1976 static void perf_counter_output(struct perf_counter *counter,
1977                                 int nmi, struct pt_regs *regs, u64 addr)
1978 {
1979         int ret;
1980         u64 record_type = counter->hw_event.record_type;
1981         struct perf_output_handle handle;
1982         struct perf_event_header header;
1983         u64 ip;
1984         struct {
1985                 u32 pid, tid;
1986         } tid_entry;
1987         struct {
1988                 u64 event;
1989                 u64 counter;
1990         } group_entry;
1991         struct perf_callchain_entry *callchain = NULL;
1992         int callchain_size = 0;
1993         u64 time;
1994         struct {
1995                 u32 cpu, reserved;
1996         } cpu_entry;
1997
1998         header.type = 0;
1999         header.size = sizeof(header);
2000
2001         header.misc = PERF_EVENT_MISC_OVERFLOW;
2002         header.misc |= user_mode(regs) ?
2003                 PERF_EVENT_MISC_USER : PERF_EVENT_MISC_KERNEL;
2004
2005         if (record_type & PERF_RECORD_IP) {
2006                 ip = instruction_pointer(regs);
2007                 header.type |= PERF_RECORD_IP;
2008                 header.size += sizeof(ip);
2009         }
2010
2011         if (record_type & PERF_RECORD_TID) {
2012                 /* namespace issues */
2013                 tid_entry.pid = current->group_leader->pid;
2014                 tid_entry.tid = current->pid;
2015
2016                 header.type |= PERF_RECORD_TID;
2017                 header.size += sizeof(tid_entry);
2018         }
2019
2020         if (record_type & PERF_RECORD_TIME) {
2021                 /*
2022                  * Maybe do better on x86 and provide cpu_clock_nmi()
2023                  */
2024                 time = sched_clock();
2025
2026                 header.type |= PERF_RECORD_TIME;
2027                 header.size += sizeof(u64);
2028         }
2029
2030         if (record_type & PERF_RECORD_ADDR) {
2031                 header.type |= PERF_RECORD_ADDR;
2032                 header.size += sizeof(u64);
2033         }
2034
2035         if (record_type & PERF_RECORD_CONFIG) {
2036                 header.type |= PERF_RECORD_CONFIG;
2037                 header.size += sizeof(u64);
2038         }
2039
2040         if (record_type & PERF_RECORD_CPU) {
2041                 header.type |= PERF_RECORD_CPU;
2042                 header.size += sizeof(cpu_entry);
2043
2044                 cpu_entry.cpu = raw_smp_processor_id();
2045         }
2046
2047         if (record_type & PERF_RECORD_GROUP) {
2048                 header.type |= PERF_RECORD_GROUP;
2049                 header.size += sizeof(u64) +
2050                         counter->nr_siblings * sizeof(group_entry);
2051         }
2052
2053         if (record_type & PERF_RECORD_CALLCHAIN) {
2054                 callchain = perf_callchain(regs);
2055
2056                 if (callchain) {
2057                         callchain_size = (1 + callchain->nr) * sizeof(u64);
2058
2059                         header.type |= PERF_RECORD_CALLCHAIN;
2060                         header.size += callchain_size;
2061                 }
2062         }
2063
2064         ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
2065         if (ret)
2066                 return;
2067
2068         perf_output_put(&handle, header);
2069
2070         if (record_type & PERF_RECORD_IP)
2071                 perf_output_put(&handle, ip);
2072
2073         if (record_type & PERF_RECORD_TID)
2074                 perf_output_put(&handle, tid_entry);
2075
2076         if (record_type & PERF_RECORD_TIME)
2077                 perf_output_put(&handle, time);
2078
2079         if (record_type & PERF_RECORD_ADDR)
2080                 perf_output_put(&handle, addr);
2081
2082         if (record_type & PERF_RECORD_CONFIG)
2083                 perf_output_put(&handle, counter->hw_event.config);
2084
2085         if (record_type & PERF_RECORD_CPU)
2086                 perf_output_put(&handle, cpu_entry);
2087
2088         /*
2089          * XXX PERF_RECORD_GROUP vs inherited counters seems difficult.
2090          */
2091         if (record_type & PERF_RECORD_GROUP) {
2092                 struct perf_counter *leader, *sub;
2093                 u64 nr = counter->nr_siblings;
2094
2095                 perf_output_put(&handle, nr);
2096
2097                 leader = counter->group_leader;
2098                 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
2099                         if (sub != counter)
2100                                 sub->pmu->read(sub);
2101
2102                         group_entry.event = sub->hw_event.config;
2103                         group_entry.counter = atomic64_read(&sub->count);
2104
2105                         perf_output_put(&handle, group_entry);
2106                 }
2107         }
2108
2109         if (callchain)
2110                 perf_output_copy(&handle, callchain, callchain_size);
2111
2112         perf_output_end(&handle);
2113 }
2114
2115 /*
2116  * comm tracking
2117  */
2118
2119 struct perf_comm_event {
2120         struct task_struct      *task;
2121         char                    *comm;
2122         int                     comm_size;
2123
2124         struct {
2125                 struct perf_event_header        header;
2126
2127                 u32                             pid;
2128                 u32                             tid;
2129         } event;
2130 };
2131
2132 static void perf_counter_comm_output(struct perf_counter *counter,
2133                                      struct perf_comm_event *comm_event)
2134 {
2135         struct perf_output_handle handle;
2136         int size = comm_event->event.header.size;
2137         int ret = perf_output_begin(&handle, counter, size, 0, 0);
2138
2139         if (ret)
2140                 return;
2141
2142         perf_output_put(&handle, comm_event->event);
2143         perf_output_copy(&handle, comm_event->comm,
2144                                    comm_event->comm_size);
2145         perf_output_end(&handle);
2146 }
2147
2148 static int perf_counter_comm_match(struct perf_counter *counter,
2149                                    struct perf_comm_event *comm_event)
2150 {
2151         if (counter->hw_event.comm &&
2152             comm_event->event.header.type == PERF_EVENT_COMM)
2153                 return 1;
2154
2155         return 0;
2156 }
2157
2158 static void perf_counter_comm_ctx(struct perf_counter_context *ctx,
2159                                   struct perf_comm_event *comm_event)
2160 {
2161         struct perf_counter *counter;
2162
2163         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2164                 return;
2165
2166         rcu_read_lock();
2167         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2168                 if (perf_counter_comm_match(counter, comm_event))
2169                         perf_counter_comm_output(counter, comm_event);
2170         }
2171         rcu_read_unlock();
2172 }
2173
2174 static void perf_counter_comm_event(struct perf_comm_event *comm_event)
2175 {
2176         struct perf_cpu_context *cpuctx;
2177         unsigned int size;
2178         char *comm = comm_event->task->comm;
2179
2180         size = ALIGN(strlen(comm)+1, sizeof(u64));
2181
2182         comm_event->comm = comm;
2183         comm_event->comm_size = size;
2184
2185         comm_event->event.header.size = sizeof(comm_event->event) + size;
2186
2187         cpuctx = &get_cpu_var(perf_cpu_context);
2188         perf_counter_comm_ctx(&cpuctx->ctx, comm_event);
2189         put_cpu_var(perf_cpu_context);
2190
2191         perf_counter_comm_ctx(&current->perf_counter_ctx, comm_event);
2192 }
2193
2194 void perf_counter_comm(struct task_struct *task)
2195 {
2196         struct perf_comm_event comm_event;
2197
2198         if (!atomic_read(&nr_comm_tracking))
2199                 return;
2200
2201         comm_event = (struct perf_comm_event){
2202                 .task   = task,
2203                 .event  = {
2204                         .header = { .type = PERF_EVENT_COMM, },
2205                         .pid    = task->group_leader->pid,
2206                         .tid    = task->pid,
2207                 },
2208         };
2209
2210         perf_counter_comm_event(&comm_event);
2211 }
2212
2213 /*
2214  * mmap tracking
2215  */
2216
2217 struct perf_mmap_event {
2218         struct file     *file;
2219         char            *file_name;
2220         int             file_size;
2221
2222         struct {
2223                 struct perf_event_header        header;
2224
2225                 u32                             pid;
2226                 u32                             tid;
2227                 u64                             start;
2228                 u64                             len;
2229                 u64                             pgoff;
2230         } event;
2231 };
2232
2233 static void perf_counter_mmap_output(struct perf_counter *counter,
2234                                      struct perf_mmap_event *mmap_event)
2235 {
2236         struct perf_output_handle handle;
2237         int size = mmap_event->event.header.size;
2238         int ret = perf_output_begin(&handle, counter, size, 0, 0);
2239
2240         if (ret)
2241                 return;
2242
2243         perf_output_put(&handle, mmap_event->event);
2244         perf_output_copy(&handle, mmap_event->file_name,
2245                                    mmap_event->file_size);
2246         perf_output_end(&handle);
2247 }
2248
2249 static int perf_counter_mmap_match(struct perf_counter *counter,
2250                                    struct perf_mmap_event *mmap_event)
2251 {
2252         if (counter->hw_event.mmap &&
2253             mmap_event->event.header.type == PERF_EVENT_MMAP)
2254                 return 1;
2255
2256         if (counter->hw_event.munmap &&
2257             mmap_event->event.header.type == PERF_EVENT_MUNMAP)
2258                 return 1;
2259
2260         return 0;
2261 }
2262
2263 static void perf_counter_mmap_ctx(struct perf_counter_context *ctx,
2264                                   struct perf_mmap_event *mmap_event)
2265 {
2266         struct perf_counter *counter;
2267
2268         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2269                 return;
2270
2271         rcu_read_lock();
2272         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2273                 if (perf_counter_mmap_match(counter, mmap_event))
2274                         perf_counter_mmap_output(counter, mmap_event);
2275         }
2276         rcu_read_unlock();
2277 }
2278
2279 static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
2280 {
2281         struct perf_cpu_context *cpuctx;
2282         struct file *file = mmap_event->file;
2283         unsigned int size;
2284         char tmp[16];
2285         char *buf = NULL;
2286         char *name;
2287
2288         if (file) {
2289                 buf = kzalloc(PATH_MAX, GFP_KERNEL);
2290                 if (!buf) {
2291                         name = strncpy(tmp, "//enomem", sizeof(tmp));
2292                         goto got_name;
2293                 }
2294                 name = d_path(&file->f_path, buf, PATH_MAX);
2295                 if (IS_ERR(name)) {
2296                         name = strncpy(tmp, "//toolong", sizeof(tmp));
2297                         goto got_name;
2298                 }
2299         } else {
2300                 name = strncpy(tmp, "//anon", sizeof(tmp));
2301                 goto got_name;
2302         }
2303
2304 got_name:
2305         size = ALIGN(strlen(name)+1, sizeof(u64));
2306
2307         mmap_event->file_name = name;
2308         mmap_event->file_size = size;
2309
2310         mmap_event->event.header.size = sizeof(mmap_event->event) + size;
2311
2312         cpuctx = &get_cpu_var(perf_cpu_context);
2313         perf_counter_mmap_ctx(&cpuctx->ctx, mmap_event);
2314         put_cpu_var(perf_cpu_context);
2315
2316         perf_counter_mmap_ctx(&current->perf_counter_ctx, mmap_event);
2317
2318         kfree(buf);
2319 }
2320
2321 void perf_counter_mmap(unsigned long addr, unsigned long len,
2322                        unsigned long pgoff, struct file *file)
2323 {
2324         struct perf_mmap_event mmap_event;
2325
2326         if (!atomic_read(&nr_mmap_tracking))
2327                 return;
2328
2329         mmap_event = (struct perf_mmap_event){
2330                 .file   = file,
2331                 .event  = {
2332                         .header = { .type = PERF_EVENT_MMAP, },
2333                         .pid    = current->group_leader->pid,
2334                         .tid    = current->pid,
2335                         .start  = addr,
2336                         .len    = len,
2337                         .pgoff  = pgoff,
2338                 },
2339         };
2340
2341         perf_counter_mmap_event(&mmap_event);
2342 }
2343
2344 void perf_counter_munmap(unsigned long addr, unsigned long len,
2345                          unsigned long pgoff, struct file *file)
2346 {
2347         struct perf_mmap_event mmap_event;
2348
2349         if (!atomic_read(&nr_munmap_tracking))
2350                 return;
2351
2352         mmap_event = (struct perf_mmap_event){
2353                 .file   = file,
2354                 .event  = {
2355                         .header = { .type = PERF_EVENT_MUNMAP, },
2356                         .pid    = current->group_leader->pid,
2357                         .tid    = current->pid,
2358                         .start  = addr,
2359                         .len    = len,
2360                         .pgoff  = pgoff,
2361                 },
2362         };
2363
2364         perf_counter_mmap_event(&mmap_event);
2365 }
2366
2367 /*
2368  * Generic counter overflow handling.
2369  */
2370
2371 int perf_counter_overflow(struct perf_counter *counter,
2372                           int nmi, struct pt_regs *regs, u64 addr)
2373 {
2374         int events = atomic_read(&counter->event_limit);
2375         int ret = 0;
2376
2377         /*
2378          * XXX event_limit might not quite work as expected on inherited
2379          * counters
2380          */
2381
2382         counter->pending_kill = POLL_IN;
2383         if (events && atomic_dec_and_test(&counter->event_limit)) {
2384                 ret = 1;
2385                 counter->pending_kill = POLL_HUP;
2386                 if (nmi) {
2387                         counter->pending_disable = 1;
2388                         perf_pending_queue(&counter->pending,
2389                                            perf_pending_counter);
2390                 } else
2391                         perf_counter_disable(counter);
2392         }
2393
2394         perf_counter_output(counter, nmi, regs, addr);
2395         return ret;
2396 }
2397
2398 /*
2399  * Generic software counter infrastructure
2400  */
2401
2402 static void perf_swcounter_update(struct perf_counter *counter)
2403 {
2404         struct hw_perf_counter *hwc = &counter->hw;
2405         u64 prev, now;
2406         s64 delta;
2407
2408 again:
2409         prev = atomic64_read(&hwc->prev_count);
2410         now = atomic64_read(&hwc->count);
2411         if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev)
2412                 goto again;
2413
2414         delta = now - prev;
2415
2416         atomic64_add(delta, &counter->count);
2417         atomic64_sub(delta, &hwc->period_left);
2418 }
2419
2420 static void perf_swcounter_set_period(struct perf_counter *counter)
2421 {
2422         struct hw_perf_counter *hwc = &counter->hw;
2423         s64 left = atomic64_read(&hwc->period_left);
2424         s64 period = hwc->irq_period;
2425
2426         if (unlikely(left <= -period)) {
2427                 left = period;
2428                 atomic64_set(&hwc->period_left, left);
2429         }
2430
2431         if (unlikely(left <= 0)) {
2432                 left += period;
2433                 atomic64_add(period, &hwc->period_left);
2434         }
2435
2436         atomic64_set(&hwc->prev_count, -left);
2437         atomic64_set(&hwc->count, -left);
2438 }
2439
2440 static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
2441 {
2442         enum hrtimer_restart ret = HRTIMER_RESTART;
2443         struct perf_counter *counter;
2444         struct pt_regs *regs;
2445
2446         counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
2447         counter->pmu->read(counter);
2448
2449         regs = get_irq_regs();
2450         /*
2451          * In case we exclude kernel IPs or are somehow not in interrupt
2452          * context, provide the next best thing, the user IP.
2453          */
2454         if ((counter->hw_event.exclude_kernel || !regs) &&
2455                         !counter->hw_event.exclude_user)
2456                 regs = task_pt_regs(current);
2457
2458         if (regs) {
2459                 if (perf_counter_overflow(counter, 0, regs, 0))
2460                         ret = HRTIMER_NORESTART;
2461         }
2462
2463         hrtimer_forward_now(hrtimer, ns_to_ktime(counter->hw.irq_period));
2464
2465         return ret;
2466 }
2467
2468 static void perf_swcounter_overflow(struct perf_counter *counter,
2469                                     int nmi, struct pt_regs *regs, u64 addr)
2470 {
2471         perf_swcounter_update(counter);
2472         perf_swcounter_set_period(counter);
2473         if (perf_counter_overflow(counter, nmi, regs, addr))
2474                 /* soft-disable the counter */
2475                 ;
2476
2477 }
2478
2479 static int perf_swcounter_match(struct perf_counter *counter,
2480                                 enum perf_event_types type,
2481                                 u32 event, struct pt_regs *regs)
2482 {
2483         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
2484                 return 0;
2485
2486         if (perf_event_raw(&counter->hw_event))
2487                 return 0;
2488
2489         if (perf_event_type(&counter->hw_event) != type)
2490                 return 0;
2491
2492         if (perf_event_id(&counter->hw_event) != event)
2493                 return 0;
2494
2495         if (counter->hw_event.exclude_user && user_mode(regs))
2496                 return 0;
2497
2498         if (counter->hw_event.exclude_kernel && !user_mode(regs))
2499                 return 0;
2500
2501         return 1;
2502 }
2503
2504 static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
2505                                int nmi, struct pt_regs *regs, u64 addr)
2506 {
2507         int neg = atomic64_add_negative(nr, &counter->hw.count);
2508         if (counter->hw.irq_period && !neg)
2509                 perf_swcounter_overflow(counter, nmi, regs, addr);
2510 }
2511
2512 static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
2513                                      enum perf_event_types type, u32 event,
2514                                      u64 nr, int nmi, struct pt_regs *regs,
2515                                      u64 addr)
2516 {
2517         struct perf_counter *counter;
2518
2519         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2520                 return;
2521
2522         rcu_read_lock();
2523         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2524                 if (perf_swcounter_match(counter, type, event, regs))
2525                         perf_swcounter_add(counter, nr, nmi, regs, addr);
2526         }
2527         rcu_read_unlock();
2528 }
2529
2530 static int *perf_swcounter_recursion_context(struct perf_cpu_context *cpuctx)
2531 {
2532         if (in_nmi())
2533                 return &cpuctx->recursion[3];
2534
2535         if (in_irq())
2536                 return &cpuctx->recursion[2];
2537
2538         if (in_softirq())
2539                 return &cpuctx->recursion[1];
2540
2541         return &cpuctx->recursion[0];
2542 }
2543
2544 static void __perf_swcounter_event(enum perf_event_types type, u32 event,
2545                                    u64 nr, int nmi, struct pt_regs *regs,
2546                                    u64 addr)
2547 {
2548         struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
2549         int *recursion = perf_swcounter_recursion_context(cpuctx);
2550
2551         if (*recursion)
2552                 goto out;
2553
2554         (*recursion)++;
2555         barrier();
2556
2557         perf_swcounter_ctx_event(&cpuctx->ctx, type, event,
2558                                  nr, nmi, regs, addr);
2559         if (cpuctx->task_ctx) {
2560                 perf_swcounter_ctx_event(cpuctx->task_ctx, type, event,
2561                                          nr, nmi, regs, addr);
2562         }
2563
2564         barrier();
2565         (*recursion)--;
2566
2567 out:
2568         put_cpu_var(perf_cpu_context);
2569 }
2570
2571 void
2572 perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
2573 {
2574         __perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, regs, addr);
2575 }
2576
2577 static void perf_swcounter_read(struct perf_counter *counter)
2578 {
2579         perf_swcounter_update(counter);
2580 }
2581
2582 static int perf_swcounter_enable(struct perf_counter *counter)
2583 {
2584         perf_swcounter_set_period(counter);
2585         return 0;
2586 }
2587
2588 static void perf_swcounter_disable(struct perf_counter *counter)
2589 {
2590         perf_swcounter_update(counter);
2591 }
2592
2593 static const struct pmu perf_ops_generic = {
2594         .enable         = perf_swcounter_enable,
2595         .disable        = perf_swcounter_disable,
2596         .read           = perf_swcounter_read,
2597 };
2598
2599 /*
2600  * Software counter: cpu wall time clock
2601  */
2602
2603 static void cpu_clock_perf_counter_update(struct perf_counter *counter)
2604 {
2605         int cpu = raw_smp_processor_id();
2606         s64 prev;
2607         u64 now;
2608
2609         now = cpu_clock(cpu);
2610         prev = atomic64_read(&counter->hw.prev_count);
2611         atomic64_set(&counter->hw.prev_count, now);
2612         atomic64_add(now - prev, &counter->count);
2613 }
2614
2615 static int cpu_clock_perf_counter_enable(struct perf_counter *counter)
2616 {
2617         struct hw_perf_counter *hwc = &counter->hw;
2618         int cpu = raw_smp_processor_id();
2619
2620         atomic64_set(&hwc->prev_count, cpu_clock(cpu));
2621         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2622         hwc->hrtimer.function = perf_swcounter_hrtimer;
2623         if (hwc->irq_period) {
2624                 __hrtimer_start_range_ns(&hwc->hrtimer,
2625                                 ns_to_ktime(hwc->irq_period), 0,
2626                                 HRTIMER_MODE_REL, 0);
2627         }
2628
2629         return 0;
2630 }
2631
2632 static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
2633 {
2634         hrtimer_cancel(&counter->hw.hrtimer);
2635         cpu_clock_perf_counter_update(counter);
2636 }
2637
2638 static void cpu_clock_perf_counter_read(struct perf_counter *counter)
2639 {
2640         cpu_clock_perf_counter_update(counter);
2641 }
2642
2643 static const struct pmu perf_ops_cpu_clock = {
2644         .enable         = cpu_clock_perf_counter_enable,
2645         .disable        = cpu_clock_perf_counter_disable,
2646         .read           = cpu_clock_perf_counter_read,
2647 };
2648
2649 /*
2650  * Software counter: task time clock
2651  */
2652
2653 static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
2654 {
2655         u64 prev;
2656         s64 delta;
2657
2658         prev = atomic64_xchg(&counter->hw.prev_count, now);
2659         delta = now - prev;
2660         atomic64_add(delta, &counter->count);
2661 }
2662
2663 static int task_clock_perf_counter_enable(struct perf_counter *counter)
2664 {
2665         struct hw_perf_counter *hwc = &counter->hw;
2666         u64 now;
2667
2668         now = counter->ctx->time;
2669
2670         atomic64_set(&hwc->prev_count, now);
2671         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2672         hwc->hrtimer.function = perf_swcounter_hrtimer;
2673         if (hwc->irq_period) {
2674                 __hrtimer_start_range_ns(&hwc->hrtimer,
2675                                 ns_to_ktime(hwc->irq_period), 0,
2676                                 HRTIMER_MODE_REL, 0);
2677         }
2678
2679         return 0;
2680 }
2681
2682 static void task_clock_perf_counter_disable(struct perf_counter *counter)
2683 {
2684         hrtimer_cancel(&counter->hw.hrtimer);
2685         task_clock_perf_counter_update(counter, counter->ctx->time);
2686
2687 }
2688
2689 static void task_clock_perf_counter_read(struct perf_counter *counter)
2690 {
2691         u64 time;
2692
2693         if (!in_nmi()) {
2694                 update_context_time(counter->ctx);
2695                 time = counter->ctx->time;
2696         } else {
2697                 u64 now = perf_clock();
2698                 u64 delta = now - counter->ctx->timestamp;
2699                 time = counter->ctx->time + delta;
2700         }
2701
2702         task_clock_perf_counter_update(counter, time);
2703 }
2704
2705 static const struct pmu perf_ops_task_clock = {
2706         .enable         = task_clock_perf_counter_enable,
2707         .disable        = task_clock_perf_counter_disable,
2708         .read           = task_clock_perf_counter_read,
2709 };
2710
2711 /*
2712  * Software counter: cpu migrations
2713  */
2714
2715 static inline u64 get_cpu_migrations(struct perf_counter *counter)
2716 {
2717         struct task_struct *curr = counter->ctx->task;
2718
2719         if (curr)
2720                 return curr->se.nr_migrations;
2721         return cpu_nr_migrations(smp_processor_id());
2722 }
2723
2724 static void cpu_migrations_perf_counter_update(struct perf_counter *counter)
2725 {
2726         u64 prev, now;
2727         s64 delta;
2728
2729         prev = atomic64_read(&counter->hw.prev_count);
2730         now = get_cpu_migrations(counter);
2731
2732         atomic64_set(&counter->hw.prev_count, now);
2733
2734         delta = now - prev;
2735
2736         atomic64_add(delta, &counter->count);
2737 }
2738
2739 static void cpu_migrations_perf_counter_read(struct perf_counter *counter)
2740 {
2741         cpu_migrations_perf_counter_update(counter);
2742 }
2743
2744 static int cpu_migrations_perf_counter_enable(struct perf_counter *counter)
2745 {
2746         if (counter->prev_state <= PERF_COUNTER_STATE_OFF)
2747                 atomic64_set(&counter->hw.prev_count,
2748                              get_cpu_migrations(counter));
2749         return 0;
2750 }
2751
2752 static void cpu_migrations_perf_counter_disable(struct perf_counter *counter)
2753 {
2754         cpu_migrations_perf_counter_update(counter);
2755 }
2756
2757 static const struct pmu perf_ops_cpu_migrations = {
2758         .enable         = cpu_migrations_perf_counter_enable,
2759         .disable        = cpu_migrations_perf_counter_disable,
2760         .read           = cpu_migrations_perf_counter_read,
2761 };
2762
2763 #ifdef CONFIG_EVENT_PROFILE
2764 void perf_tpcounter_event(int event_id)
2765 {
2766         struct pt_regs *regs = get_irq_regs();
2767
2768         if (!regs)
2769                 regs = task_pt_regs(current);
2770
2771         __perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, regs, 0);
2772 }
2773 EXPORT_SYMBOL_GPL(perf_tpcounter_event);
2774
2775 extern int ftrace_profile_enable(int);
2776 extern void ftrace_profile_disable(int);
2777
2778 static void tp_perf_counter_destroy(struct perf_counter *counter)
2779 {
2780         ftrace_profile_disable(perf_event_id(&counter->hw_event));
2781 }
2782
2783 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
2784 {
2785         int event_id = perf_event_id(&counter->hw_event);
2786         int ret;
2787
2788         ret = ftrace_profile_enable(event_id);
2789         if (ret)
2790                 return NULL;
2791
2792         counter->destroy = tp_perf_counter_destroy;
2793         counter->hw.irq_period = counter->hw_event.irq_period;
2794
2795         return &perf_ops_generic;
2796 }
2797 #else
2798 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
2799 {
2800         return NULL;
2801 }
2802 #endif
2803
2804 static const struct pmu *sw_perf_counter_init(struct perf_counter *counter)
2805 {
2806         struct perf_counter_hw_event *hw_event = &counter->hw_event;
2807         const struct pmu *pmu = NULL;
2808         struct hw_perf_counter *hwc = &counter->hw;
2809
2810         /*
2811          * Software counters (currently) can't in general distinguish
2812          * between user, kernel and hypervisor events.
2813          * However, context switches and cpu migrations are considered
2814          * to be kernel events, and page faults are never hypervisor
2815          * events.
2816          */
2817         switch (perf_event_id(&counter->hw_event)) {
2818         case PERF_COUNT_CPU_CLOCK:
2819                 pmu = &perf_ops_cpu_clock;
2820
2821                 if (hw_event->irq_period && hw_event->irq_period < 10000)
2822                         hw_event->irq_period = 10000;
2823                 break;
2824         case PERF_COUNT_TASK_CLOCK:
2825                 /*
2826                  * If the user instantiates this as a per-cpu counter,
2827                  * use the cpu_clock counter instead.
2828                  */
2829                 if (counter->ctx->task)
2830                         pmu = &perf_ops_task_clock;
2831                 else
2832                         pmu = &perf_ops_cpu_clock;
2833
2834                 if (hw_event->irq_period && hw_event->irq_period < 10000)
2835                         hw_event->irq_period = 10000;
2836                 break;
2837         case PERF_COUNT_PAGE_FAULTS:
2838         case PERF_COUNT_PAGE_FAULTS_MIN:
2839         case PERF_COUNT_PAGE_FAULTS_MAJ:
2840         case PERF_COUNT_CONTEXT_SWITCHES:
2841                 pmu = &perf_ops_generic;
2842                 break;
2843         case PERF_COUNT_CPU_MIGRATIONS:
2844                 if (!counter->hw_event.exclude_kernel)
2845                         pmu = &perf_ops_cpu_migrations;
2846                 break;
2847         }
2848
2849         if (pmu)
2850                 hwc->irq_period = hw_event->irq_period;
2851
2852         return pmu;
2853 }
2854
2855 /*
2856  * Allocate and initialize a counter structure
2857  */
2858 static struct perf_counter *
2859 perf_counter_alloc(struct perf_counter_hw_event *hw_event,
2860                    int cpu,
2861                    struct perf_counter_context *ctx,
2862                    struct perf_counter *group_leader,
2863                    gfp_t gfpflags)
2864 {
2865         const struct pmu *pmu;
2866         struct perf_counter *counter;
2867         long err;
2868
2869         counter = kzalloc(sizeof(*counter), gfpflags);
2870         if (!counter)
2871                 return ERR_PTR(-ENOMEM);
2872
2873         /*
2874          * Single counters are their own group leaders, with an
2875          * empty sibling list:
2876          */
2877         if (!group_leader)
2878                 group_leader = counter;
2879
2880         mutex_init(&counter->mutex);
2881         INIT_LIST_HEAD(&counter->list_entry);
2882         INIT_LIST_HEAD(&counter->event_entry);
2883         INIT_LIST_HEAD(&counter->sibling_list);
2884         init_waitqueue_head(&counter->waitq);
2885
2886         mutex_init(&counter->mmap_mutex);
2887
2888         INIT_LIST_HEAD(&counter->child_list);
2889
2890         counter->cpu                    = cpu;
2891         counter->hw_event               = *hw_event;
2892         counter->group_leader           = group_leader;
2893         counter->pmu                    = NULL;
2894         counter->ctx                    = ctx;
2895
2896         counter->state = PERF_COUNTER_STATE_INACTIVE;
2897         if (hw_event->disabled)
2898                 counter->state = PERF_COUNTER_STATE_OFF;
2899
2900         pmu = NULL;
2901
2902         /*
2903          * we currently do not support PERF_RECORD_GROUP on inherited counters
2904          */
2905         if (hw_event->inherit && (hw_event->record_type & PERF_RECORD_GROUP))
2906                 goto done;
2907
2908         if (perf_event_raw(hw_event)) {
2909                 pmu = hw_perf_counter_init(counter);
2910                 goto done;
2911         }
2912
2913         switch (perf_event_type(hw_event)) {
2914         case PERF_TYPE_HARDWARE:
2915                 pmu = hw_perf_counter_init(counter);
2916                 break;
2917
2918         case PERF_TYPE_SOFTWARE:
2919                 pmu = sw_perf_counter_init(counter);
2920                 break;
2921
2922         case PERF_TYPE_TRACEPOINT:
2923                 pmu = tp_perf_counter_init(counter);
2924                 break;
2925         }
2926 done:
2927         err = 0;
2928         if (!pmu)
2929                 err = -EINVAL;
2930         else if (IS_ERR(pmu))
2931                 err = PTR_ERR(pmu);
2932
2933         if (err) {
2934                 kfree(counter);
2935                 return ERR_PTR(err);
2936         }
2937
2938         counter->pmu = pmu;
2939
2940         atomic_inc(&nr_counters);
2941         if (counter->hw_event.mmap)
2942                 atomic_inc(&nr_mmap_tracking);
2943         if (counter->hw_event.munmap)
2944                 atomic_inc(&nr_munmap_tracking);
2945         if (counter->hw_event.comm)
2946                 atomic_inc(&nr_comm_tracking);
2947
2948         return counter;
2949 }
2950
2951 /**
2952  * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
2953  *
2954  * @hw_event_uptr:      event type attributes for monitoring/sampling
2955  * @pid:                target pid
2956  * @cpu:                target cpu
2957  * @group_fd:           group leader counter fd
2958  */
2959 SYSCALL_DEFINE5(perf_counter_open,
2960                 const struct perf_counter_hw_event __user *, hw_event_uptr,
2961                 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
2962 {
2963         struct perf_counter *counter, *group_leader;
2964         struct perf_counter_hw_event hw_event;
2965         struct perf_counter_context *ctx;
2966         struct file *counter_file = NULL;
2967         struct file *group_file = NULL;
2968         int fput_needed = 0;
2969         int fput_needed2 = 0;
2970         int ret;
2971
2972         /* for future expandability... */
2973         if (flags)
2974                 return -EINVAL;
2975
2976         if (copy_from_user(&hw_event, hw_event_uptr, sizeof(hw_event)) != 0)
2977                 return -EFAULT;
2978
2979         /*
2980          * Get the target context (task or percpu):
2981          */
2982         ctx = find_get_context(pid, cpu);
2983         if (IS_ERR(ctx))
2984                 return PTR_ERR(ctx);
2985
2986         /*
2987          * Look up the group leader (we will attach this counter to it):
2988          */
2989         group_leader = NULL;
2990         if (group_fd != -1) {
2991                 ret = -EINVAL;
2992                 group_file = fget_light(group_fd, &fput_needed);
2993                 if (!group_file)
2994                         goto err_put_context;
2995                 if (group_file->f_op != &perf_fops)
2996                         goto err_put_context;
2997
2998                 group_leader = group_file->private_data;
2999                 /*
3000                  * Do not allow a recursive hierarchy (this new sibling
3001                  * becoming part of another group-sibling):
3002                  */
3003                 if (group_leader->group_leader != group_leader)
3004                         goto err_put_context;
3005                 /*
3006                  * Do not allow to attach to a group in a different
3007                  * task or CPU context:
3008                  */
3009                 if (group_leader->ctx != ctx)
3010                         goto err_put_context;
3011                 /*
3012                  * Only a group leader can be exclusive or pinned
3013                  */
3014                 if (hw_event.exclusive || hw_event.pinned)
3015                         goto err_put_context;
3016         }
3017
3018         counter = perf_counter_alloc(&hw_event, cpu, ctx, group_leader,
3019                                      GFP_KERNEL);
3020         ret = PTR_ERR(counter);
3021         if (IS_ERR(counter))
3022                 goto err_put_context;
3023
3024         ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
3025         if (ret < 0)
3026                 goto err_free_put_context;
3027
3028         counter_file = fget_light(ret, &fput_needed2);
3029         if (!counter_file)
3030                 goto err_free_put_context;
3031
3032         counter->filp = counter_file;
3033         mutex_lock(&ctx->mutex);
3034         perf_install_in_context(ctx, counter, cpu);
3035         mutex_unlock(&ctx->mutex);
3036
3037         fput_light(counter_file, fput_needed2);
3038
3039 out_fput:
3040         fput_light(group_file, fput_needed);
3041
3042         return ret;
3043
3044 err_free_put_context:
3045         kfree(counter);
3046
3047 err_put_context:
3048         put_context(ctx);
3049
3050         goto out_fput;
3051 }
3052
3053 /*
3054  * Initialize the perf_counter context in a task_struct:
3055  */
3056 static void
3057 __perf_counter_init_context(struct perf_counter_context *ctx,
3058                             struct task_struct *task)
3059 {
3060         memset(ctx, 0, sizeof(*ctx));
3061         spin_lock_init(&ctx->lock);
3062         mutex_init(&ctx->mutex);
3063         INIT_LIST_HEAD(&ctx->counter_list);
3064         INIT_LIST_HEAD(&ctx->event_list);
3065         ctx->task = task;
3066 }
3067
3068 /*
3069  * inherit a counter from parent task to child task:
3070  */
3071 static struct perf_counter *
3072 inherit_counter(struct perf_counter *parent_counter,
3073               struct task_struct *parent,
3074               struct perf_counter_context *parent_ctx,
3075               struct task_struct *child,
3076               struct perf_counter *group_leader,
3077               struct perf_counter_context *child_ctx)
3078 {
3079         struct perf_counter *child_counter;
3080
3081         /*
3082          * Instead of creating recursive hierarchies of counters,
3083          * we link inherited counters back to the original parent,
3084          * which has a filp for sure, which we use as the reference
3085          * count:
3086          */
3087         if (parent_counter->parent)
3088                 parent_counter = parent_counter->parent;
3089
3090         child_counter = perf_counter_alloc(&parent_counter->hw_event,
3091                                            parent_counter->cpu, child_ctx,
3092                                            group_leader, GFP_KERNEL);
3093         if (IS_ERR(child_counter))
3094                 return child_counter;
3095
3096         /*
3097          * Link it up in the child's context:
3098          */
3099         child_counter->task = child;
3100         add_counter_to_ctx(child_counter, child_ctx);
3101
3102         child_counter->parent = parent_counter;
3103         /*
3104          * inherit into child's child as well:
3105          */
3106         child_counter->hw_event.inherit = 1;
3107
3108         /*
3109          * Get a reference to the parent filp - we will fput it
3110          * when the child counter exits. This is safe to do because
3111          * we are in the parent and we know that the filp still
3112          * exists and has a nonzero count:
3113          */
3114         atomic_long_inc(&parent_counter->filp->f_count);
3115
3116         /*
3117          * Link this into the parent counter's child list
3118          */
3119         mutex_lock(&parent_counter->mutex);
3120         list_add_tail(&child_counter->child_list, &parent_counter->child_list);
3121
3122         /*
3123          * Make the child state follow the state of the parent counter,
3124          * not its hw_event.disabled bit.  We hold the parent's mutex,
3125          * so we won't race with perf_counter_{en,dis}able_family.
3126          */
3127         if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
3128                 child_counter->state = PERF_COUNTER_STATE_INACTIVE;
3129         else
3130                 child_counter->state = PERF_COUNTER_STATE_OFF;
3131
3132         mutex_unlock(&parent_counter->mutex);
3133
3134         return child_counter;
3135 }
3136
3137 static int inherit_group(struct perf_counter *parent_counter,
3138               struct task_struct *parent,
3139               struct perf_counter_context *parent_ctx,
3140               struct task_struct *child,
3141               struct perf_counter_context *child_ctx)
3142 {
3143         struct perf_counter *leader;
3144         struct perf_counter *sub;
3145         struct perf_counter *child_ctr;
3146
3147         leader = inherit_counter(parent_counter, parent, parent_ctx,
3148                                  child, NULL, child_ctx);
3149         if (IS_ERR(leader))
3150                 return PTR_ERR(leader);
3151         list_for_each_entry(sub, &parent_counter->sibling_list, list_entry) {
3152                 child_ctr = inherit_counter(sub, parent, parent_ctx,
3153                                             child, leader, child_ctx);
3154                 if (IS_ERR(child_ctr))
3155                         return PTR_ERR(child_ctr);
3156         }
3157         return 0;
3158 }
3159
3160 static void sync_child_counter(struct perf_counter *child_counter,
3161                                struct perf_counter *parent_counter)
3162 {
3163         u64 parent_val, child_val;
3164
3165         parent_val = atomic64_read(&parent_counter->count);
3166         child_val = atomic64_read(&child_counter->count);
3167
3168         /*
3169          * Add back the child's count to the parent's count:
3170          */
3171         atomic64_add(child_val, &parent_counter->count);
3172         atomic64_add(child_counter->total_time_enabled,
3173                      &parent_counter->child_total_time_enabled);
3174         atomic64_add(child_counter->total_time_running,
3175                      &parent_counter->child_total_time_running);
3176
3177         /*
3178          * Remove this counter from the parent's list
3179          */
3180         mutex_lock(&parent_counter->mutex);
3181         list_del_init(&child_counter->child_list);
3182         mutex_unlock(&parent_counter->mutex);
3183
3184         /*
3185          * Release the parent counter, if this was the last
3186          * reference to it.
3187          */
3188         fput(parent_counter->filp);
3189 }
3190
3191 static void
3192 __perf_counter_exit_task(struct task_struct *child,
3193                          struct perf_counter *child_counter,
3194                          struct perf_counter_context *child_ctx)
3195 {
3196         struct perf_counter *parent_counter;
3197         struct perf_counter *sub, *tmp;
3198
3199         /*
3200          * If we do not self-reap then we have to wait for the
3201          * child task to unschedule (it will happen for sure),
3202          * so that its counter is at its final count. (This
3203          * condition triggers rarely - child tasks usually get
3204          * off their CPU before the parent has a chance to
3205          * get this far into the reaping action)
3206          */
3207         if (child != current) {
3208                 wait_task_inactive(child, 0);
3209                 list_del_init(&child_counter->list_entry);
3210                 update_counter_times(child_counter);
3211         } else {
3212                 struct perf_cpu_context *cpuctx;
3213                 unsigned long flags;
3214
3215                 /*
3216                  * Disable and unlink this counter.
3217                  *
3218                  * Be careful about zapping the list - IRQ/NMI context
3219                  * could still be processing it:
3220                  */
3221                 local_irq_save(flags);
3222                 perf_disable();
3223
3224                 cpuctx = &__get_cpu_var(perf_cpu_context);
3225
3226                 group_sched_out(child_counter, cpuctx, child_ctx);
3227                 update_counter_times(child_counter);
3228
3229                 list_del_init(&child_counter->list_entry);
3230
3231                 child_ctx->nr_counters--;
3232
3233                 perf_enable();
3234                 local_irq_restore(flags);
3235         }
3236
3237         parent_counter = child_counter->parent;
3238         /*
3239          * It can happen that parent exits first, and has counters
3240          * that are still around due to the child reference. These
3241          * counters need to be zapped - but otherwise linger.
3242          */
3243         if (parent_counter) {
3244                 sync_child_counter(child_counter, parent_counter);
3245                 list_for_each_entry_safe(sub, tmp, &child_counter->sibling_list,
3246                                          list_entry) {
3247                         if (sub->parent) {
3248                                 sync_child_counter(sub, sub->parent);
3249                                 free_counter(sub);
3250                         }
3251                 }
3252                 free_counter(child_counter);
3253         }
3254 }
3255
3256 /*
3257  * When a child task exits, feed back counter values to parent counters.
3258  *
3259  * Note: we may be running in child context, but the PID is not hashed
3260  * anymore so new counters will not be added.
3261  */
3262 void perf_counter_exit_task(struct task_struct *child)
3263 {
3264         struct perf_counter *child_counter, *tmp;
3265         struct perf_counter_context *child_ctx;
3266
3267         child_ctx = &child->perf_counter_ctx;
3268
3269         if (likely(!child_ctx->nr_counters))
3270                 return;
3271
3272         list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
3273                                  list_entry)
3274                 __perf_counter_exit_task(child, child_counter, child_ctx);
3275 }
3276
3277 /*
3278  * Initialize the perf_counter context in task_struct
3279  */
3280 void perf_counter_init_task(struct task_struct *child)
3281 {
3282         struct perf_counter_context *child_ctx, *parent_ctx;
3283         struct perf_counter *counter;
3284         struct task_struct *parent = current;
3285
3286         child_ctx  =  &child->perf_counter_ctx;
3287         parent_ctx = &parent->perf_counter_ctx;
3288
3289         __perf_counter_init_context(child_ctx, child);
3290
3291         /*
3292          * This is executed from the parent task context, so inherit
3293          * counters that have been marked for cloning:
3294          */
3295
3296         if (likely(!parent_ctx->nr_counters))
3297                 return;
3298
3299         /*
3300          * Lock the parent list. No need to lock the child - not PID
3301          * hashed yet and not running, so nobody can access it.
3302          */
3303         mutex_lock(&parent_ctx->mutex);
3304
3305         /*
3306          * We dont have to disable NMIs - we are only looking at
3307          * the list, not manipulating it:
3308          */
3309         list_for_each_entry(counter, &parent_ctx->counter_list, list_entry) {
3310                 if (!counter->hw_event.inherit)
3311                         continue;
3312
3313                 if (inherit_group(counter, parent,
3314                                   parent_ctx, child, child_ctx))
3315                         break;
3316         }
3317
3318         mutex_unlock(&parent_ctx->mutex);
3319 }
3320
3321 static void __cpuinit perf_counter_init_cpu(int cpu)
3322 {
3323         struct perf_cpu_context *cpuctx;
3324
3325         cpuctx = &per_cpu(perf_cpu_context, cpu);
3326         __perf_counter_init_context(&cpuctx->ctx, NULL);
3327
3328         spin_lock(&perf_resource_lock);
3329         cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
3330         spin_unlock(&perf_resource_lock);
3331
3332         hw_perf_counter_setup(cpu);
3333 }
3334
3335 #ifdef CONFIG_HOTPLUG_CPU
3336 static void __perf_counter_exit_cpu(void *info)
3337 {
3338         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
3339         struct perf_counter_context *ctx = &cpuctx->ctx;
3340         struct perf_counter *counter, *tmp;
3341
3342         list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
3343                 __perf_counter_remove_from_context(counter);
3344 }
3345 static void perf_counter_exit_cpu(int cpu)
3346 {
3347         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
3348         struct perf_counter_context *ctx = &cpuctx->ctx;
3349
3350         mutex_lock(&ctx->mutex);
3351         smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
3352         mutex_unlock(&ctx->mutex);
3353 }
3354 #else
3355 static inline void perf_counter_exit_cpu(int cpu) { }
3356 #endif
3357
3358 static int __cpuinit
3359 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
3360 {
3361         unsigned int cpu = (long)hcpu;
3362
3363         switch (action) {
3364
3365         case CPU_UP_PREPARE:
3366         case CPU_UP_PREPARE_FROZEN:
3367                 perf_counter_init_cpu(cpu);
3368                 break;
3369
3370         case CPU_DOWN_PREPARE:
3371         case CPU_DOWN_PREPARE_FROZEN:
3372                 perf_counter_exit_cpu(cpu);
3373                 break;
3374
3375         default:
3376                 break;
3377         }
3378
3379         return NOTIFY_OK;
3380 }
3381
3382 static struct notifier_block __cpuinitdata perf_cpu_nb = {
3383         .notifier_call          = perf_cpu_notify,
3384 };
3385
3386 void __init perf_counter_init(void)
3387 {
3388         perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
3389                         (void *)(long)smp_processor_id());
3390         register_cpu_notifier(&perf_cpu_nb);
3391 }
3392
3393 static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
3394 {
3395         return sprintf(buf, "%d\n", perf_reserved_percpu);
3396 }
3397
3398 static ssize_t
3399 perf_set_reserve_percpu(struct sysdev_class *class,
3400                         const char *buf,
3401                         size_t count)
3402 {
3403         struct perf_cpu_context *cpuctx;
3404         unsigned long val;
3405         int err, cpu, mpt;
3406
3407         err = strict_strtoul(buf, 10, &val);
3408         if (err)
3409                 return err;
3410         if (val > perf_max_counters)
3411                 return -EINVAL;
3412
3413         spin_lock(&perf_resource_lock);
3414         perf_reserved_percpu = val;
3415         for_each_online_cpu(cpu) {
3416                 cpuctx = &per_cpu(perf_cpu_context, cpu);
3417                 spin_lock_irq(&cpuctx->ctx.lock);
3418                 mpt = min(perf_max_counters - cpuctx->ctx.nr_counters,
3419                           perf_max_counters - perf_reserved_percpu);
3420                 cpuctx->max_pertask = mpt;
3421                 spin_unlock_irq(&cpuctx->ctx.lock);
3422         }
3423         spin_unlock(&perf_resource_lock);
3424
3425         return count;
3426 }
3427
3428 static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
3429 {
3430         return sprintf(buf, "%d\n", perf_overcommit);
3431 }
3432
3433 static ssize_t
3434 perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
3435 {
3436         unsigned long val;
3437         int err;
3438
3439         err = strict_strtoul(buf, 10, &val);
3440         if (err)
3441                 return err;
3442         if (val > 1)
3443                 return -EINVAL;
3444
3445         spin_lock(&perf_resource_lock);
3446         perf_overcommit = val;
3447         spin_unlock(&perf_resource_lock);
3448
3449         return count;
3450 }
3451
3452 static SYSDEV_CLASS_ATTR(
3453                                 reserve_percpu,
3454                                 0644,
3455                                 perf_show_reserve_percpu,
3456                                 perf_set_reserve_percpu
3457                         );
3458
3459 static SYSDEV_CLASS_ATTR(
3460                                 overcommit,
3461                                 0644,
3462                                 perf_show_overcommit,
3463                                 perf_set_overcommit
3464                         );
3465
3466 static struct attribute *perfclass_attrs[] = {
3467         &attr_reserve_percpu.attr,
3468         &attr_overcommit.attr,
3469         NULL
3470 };
3471
3472 static struct attribute_group perfclass_attr_group = {
3473         .attrs                  = perfclass_attrs,
3474         .name                   = "perf_counters",
3475 };
3476
3477 static int __init perf_counter_sysfs_init(void)
3478 {
3479         return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
3480                                   &perfclass_attr_group);
3481 }
3482 device_initcall(perf_counter_sysfs_init);