2 * Performance counter core code
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
9 * For licensing details see kernel-base/COPYING
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/file.h>
17 #include <linux/poll.h>
18 #include <linux/sysfs.h>
19 #include <linux/ptrace.h>
20 #include <linux/percpu.h>
21 #include <linux/vmstat.h>
22 #include <linux/hardirq.h>
23 #include <linux/rculist.h>
24 #include <linux/uaccess.h>
25 #include <linux/syscalls.h>
26 #include <linux/anon_inodes.h>
27 #include <linux/kernel_stat.h>
28 #include <linux/perf_counter.h>
29 #include <linux/dcache.h>
31 #include <asm/irq_regs.h>
34 * Each CPU has a list of per CPU counters:
36 DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
38 int perf_max_counters __read_mostly = 1;
39 static int perf_reserved_percpu __read_mostly;
40 static int perf_overcommit __read_mostly = 1;
42 static atomic_t nr_counters __read_mostly;
43 static atomic_t nr_mmap_tracking __read_mostly;
44 static atomic_t nr_munmap_tracking __read_mostly;
45 static atomic_t nr_comm_tracking __read_mostly;
47 int sysctl_perf_counter_priv __read_mostly; /* do we need to be privileged */
48 int sysctl_perf_counter_mlock __read_mostly = 512; /* 'free' kb per user */
49 int sysctl_perf_counter_limit __read_mostly = 100000; /* max NMIs per second */
52 * Lock for (sysadmin-configurable) counter reservations:
54 static DEFINE_SPINLOCK(perf_resource_lock);
57 * Architecture provided APIs - weak aliases:
59 extern __weak const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
64 void __weak hw_perf_disable(void) { barrier(); }
65 void __weak hw_perf_enable(void) { barrier(); }
67 void __weak hw_perf_counter_setup(int cpu) { barrier(); }
68 int __weak hw_perf_group_sched_in(struct perf_counter *group_leader,
69 struct perf_cpu_context *cpuctx,
70 struct perf_counter_context *ctx, int cpu)
75 void __weak perf_counter_print_debug(void) { }
77 static DEFINE_PER_CPU(int, disable_count);
79 void __perf_disable(void)
81 __get_cpu_var(disable_count)++;
84 bool __perf_enable(void)
86 return !--__get_cpu_var(disable_count);
89 void perf_disable(void)
95 void perf_enable(void)
101 static void get_ctx(struct perf_counter_context *ctx)
103 atomic_inc(&ctx->refcount);
106 static void free_ctx(struct rcu_head *head)
108 struct perf_counter_context *ctx;
110 ctx = container_of(head, struct perf_counter_context, rcu_head);
114 static void put_ctx(struct perf_counter_context *ctx)
116 if (atomic_dec_and_test(&ctx->refcount)) {
118 put_ctx(ctx->parent_ctx);
120 put_task_struct(ctx->task);
121 call_rcu(&ctx->rcu_head, free_ctx);
126 * Add a counter from the lists for its context.
127 * Must be called with ctx->mutex and ctx->lock held.
130 list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
132 struct perf_counter *group_leader = counter->group_leader;
135 * Depending on whether it is a standalone or sibling counter,
136 * add it straight to the context's counter list, or to the group
137 * leader's sibling list:
139 if (group_leader == counter)
140 list_add_tail(&counter->list_entry, &ctx->counter_list);
142 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
143 group_leader->nr_siblings++;
146 list_add_rcu(&counter->event_entry, &ctx->event_list);
151 * Remove a counter from the lists for its context.
152 * Must be called with ctx->mutex and ctx->lock held.
155 list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
157 struct perf_counter *sibling, *tmp;
159 if (list_empty(&counter->list_entry))
163 list_del_init(&counter->list_entry);
164 list_del_rcu(&counter->event_entry);
166 if (counter->group_leader != counter)
167 counter->group_leader->nr_siblings--;
170 * If this was a group counter with sibling counters then
171 * upgrade the siblings to singleton counters by adding them
172 * to the context list directly:
174 list_for_each_entry_safe(sibling, tmp,
175 &counter->sibling_list, list_entry) {
177 list_move_tail(&sibling->list_entry, &ctx->counter_list);
178 sibling->group_leader = sibling;
183 counter_sched_out(struct perf_counter *counter,
184 struct perf_cpu_context *cpuctx,
185 struct perf_counter_context *ctx)
187 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
190 counter->state = PERF_COUNTER_STATE_INACTIVE;
191 counter->tstamp_stopped = ctx->time;
192 counter->pmu->disable(counter);
195 if (!is_software_counter(counter))
196 cpuctx->active_oncpu--;
198 if (counter->hw_event.exclusive || !cpuctx->active_oncpu)
199 cpuctx->exclusive = 0;
203 group_sched_out(struct perf_counter *group_counter,
204 struct perf_cpu_context *cpuctx,
205 struct perf_counter_context *ctx)
207 struct perf_counter *counter;
209 if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
212 counter_sched_out(group_counter, cpuctx, ctx);
215 * Schedule out siblings (if any):
217 list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
218 counter_sched_out(counter, cpuctx, ctx);
220 if (group_counter->hw_event.exclusive)
221 cpuctx->exclusive = 0;
225 * Cross CPU call to remove a performance counter
227 * We disable the counter on the hardware level first. After that we
228 * remove it from the context list.
230 static void __perf_counter_remove_from_context(void *info)
232 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
233 struct perf_counter *counter = info;
234 struct perf_counter_context *ctx = counter->ctx;
237 local_irq_save(flags);
239 * If this is a task context, we need to check whether it is
240 * the current task context of this cpu. If not it has been
241 * scheduled out before the smp call arrived.
243 if (ctx->task && cpuctx->task_ctx != ctx) {
244 local_irq_restore(flags);
248 spin_lock(&ctx->lock);
250 * Protect the list operation against NMI by disabling the
251 * counters on a global level.
255 counter_sched_out(counter, cpuctx, ctx);
257 list_del_counter(counter, ctx);
261 * Allow more per task counters with respect to the
264 cpuctx->max_pertask =
265 min(perf_max_counters - ctx->nr_counters,
266 perf_max_counters - perf_reserved_percpu);
270 spin_unlock_irqrestore(&ctx->lock, flags);
275 * Remove the counter from a task's (or a CPU's) list of counters.
277 * Must be called with ctx->mutex held.
279 * CPU counters are removed with a smp call. For task counters we only
280 * call when the task is on a CPU.
282 * If counter->ctx is a cloned context, callers must make sure that
283 * every task struct that counter->ctx->task could possibly point to
284 * remains valid. This is OK when called from perf_release since
285 * that only calls us on the top-level context, which can't be a clone.
286 * When called from perf_counter_exit_task, it's OK because the
287 * context has been detached from its task.
289 static void perf_counter_remove_from_context(struct perf_counter *counter)
291 struct perf_counter_context *ctx = counter->ctx;
292 struct task_struct *task = ctx->task;
296 * Per cpu counters are removed via an smp call and
297 * the removal is always sucessful.
299 smp_call_function_single(counter->cpu,
300 __perf_counter_remove_from_context,
306 task_oncpu_function_call(task, __perf_counter_remove_from_context,
309 spin_lock_irq(&ctx->lock);
311 * If the context is active we need to retry the smp call.
313 if (ctx->nr_active && !list_empty(&counter->list_entry)) {
314 spin_unlock_irq(&ctx->lock);
319 * The lock prevents that this context is scheduled in so we
320 * can remove the counter safely, if the call above did not
323 if (!list_empty(&counter->list_entry)) {
324 list_del_counter(counter, ctx);
326 spin_unlock_irq(&ctx->lock);
329 static inline u64 perf_clock(void)
331 return cpu_clock(smp_processor_id());
335 * Update the record of the current time in a context.
337 static void update_context_time(struct perf_counter_context *ctx)
339 u64 now = perf_clock();
341 ctx->time += now - ctx->timestamp;
342 ctx->timestamp = now;
346 * Update the total_time_enabled and total_time_running fields for a counter.
348 static void update_counter_times(struct perf_counter *counter)
350 struct perf_counter_context *ctx = counter->ctx;
353 if (counter->state < PERF_COUNTER_STATE_INACTIVE)
356 counter->total_time_enabled = ctx->time - counter->tstamp_enabled;
358 if (counter->state == PERF_COUNTER_STATE_INACTIVE)
359 run_end = counter->tstamp_stopped;
363 counter->total_time_running = run_end - counter->tstamp_running;
367 * Update total_time_enabled and total_time_running for all counters in a group.
369 static void update_group_times(struct perf_counter *leader)
371 struct perf_counter *counter;
373 update_counter_times(leader);
374 list_for_each_entry(counter, &leader->sibling_list, list_entry)
375 update_counter_times(counter);
379 * Cross CPU call to disable a performance counter
381 static void __perf_counter_disable(void *info)
383 struct perf_counter *counter = info;
384 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
385 struct perf_counter_context *ctx = counter->ctx;
388 local_irq_save(flags);
390 * If this is a per-task counter, need to check whether this
391 * counter's task is the current task on this cpu.
393 if (ctx->task && cpuctx->task_ctx != ctx) {
394 local_irq_restore(flags);
398 spin_lock(&ctx->lock);
401 * If the counter is on, turn it off.
402 * If it is in error state, leave it in error state.
404 if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
405 update_context_time(ctx);
406 update_counter_times(counter);
407 if (counter == counter->group_leader)
408 group_sched_out(counter, cpuctx, ctx);
410 counter_sched_out(counter, cpuctx, ctx);
411 counter->state = PERF_COUNTER_STATE_OFF;
414 spin_unlock_irqrestore(&ctx->lock, flags);
420 * If counter->ctx is a cloned context, callers must make sure that
421 * every task struct that counter->ctx->task could possibly point to
422 * remains valid. This condition is satisifed when called through
423 * perf_counter_for_each_child or perf_counter_for_each because they
424 * hold the top-level counter's child_mutex, so any descendant that
425 * goes to exit will block in sync_child_counter.
426 * When called from perf_pending_counter it's OK because counter->ctx
427 * is the current context on this CPU and preemption is disabled,
428 * hence we can't get into perf_counter_task_sched_out for this context.
430 static void perf_counter_disable(struct perf_counter *counter)
432 struct perf_counter_context *ctx = counter->ctx;
433 struct task_struct *task = ctx->task;
437 * Disable the counter on the cpu that it's on
439 smp_call_function_single(counter->cpu, __perf_counter_disable,
445 task_oncpu_function_call(task, __perf_counter_disable, counter);
447 spin_lock_irq(&ctx->lock);
449 * If the counter is still active, we need to retry the cross-call.
451 if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
452 spin_unlock_irq(&ctx->lock);
457 * Since we have the lock this context can't be scheduled
458 * in, so we can change the state safely.
460 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
461 update_counter_times(counter);
462 counter->state = PERF_COUNTER_STATE_OFF;
465 spin_unlock_irq(&ctx->lock);
469 counter_sched_in(struct perf_counter *counter,
470 struct perf_cpu_context *cpuctx,
471 struct perf_counter_context *ctx,
474 if (counter->state <= PERF_COUNTER_STATE_OFF)
477 counter->state = PERF_COUNTER_STATE_ACTIVE;
478 counter->oncpu = cpu; /* TODO: put 'cpu' into cpuctx->cpu */
480 * The new state must be visible before we turn it on in the hardware:
484 if (counter->pmu->enable(counter)) {
485 counter->state = PERF_COUNTER_STATE_INACTIVE;
490 counter->tstamp_running += ctx->time - counter->tstamp_stopped;
492 if (!is_software_counter(counter))
493 cpuctx->active_oncpu++;
496 if (counter->hw_event.exclusive)
497 cpuctx->exclusive = 1;
503 group_sched_in(struct perf_counter *group_counter,
504 struct perf_cpu_context *cpuctx,
505 struct perf_counter_context *ctx,
508 struct perf_counter *counter, *partial_group;
511 if (group_counter->state == PERF_COUNTER_STATE_OFF)
514 ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
516 return ret < 0 ? ret : 0;
518 group_counter->prev_state = group_counter->state;
519 if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
523 * Schedule in siblings as one group (if any):
525 list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
526 counter->prev_state = counter->state;
527 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
528 partial_group = counter;
537 * Groups can be scheduled in as one unit only, so undo any
538 * partial group before returning:
540 list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
541 if (counter == partial_group)
543 counter_sched_out(counter, cpuctx, ctx);
545 counter_sched_out(group_counter, cpuctx, ctx);
551 * Return 1 for a group consisting entirely of software counters,
552 * 0 if the group contains any hardware counters.
554 static int is_software_only_group(struct perf_counter *leader)
556 struct perf_counter *counter;
558 if (!is_software_counter(leader))
561 list_for_each_entry(counter, &leader->sibling_list, list_entry)
562 if (!is_software_counter(counter))
569 * Work out whether we can put this counter group on the CPU now.
571 static int group_can_go_on(struct perf_counter *counter,
572 struct perf_cpu_context *cpuctx,
576 * Groups consisting entirely of software counters can always go on.
578 if (is_software_only_group(counter))
581 * If an exclusive group is already on, no other hardware
582 * counters can go on.
584 if (cpuctx->exclusive)
587 * If this group is exclusive and there are already
588 * counters on the CPU, it can't go on.
590 if (counter->hw_event.exclusive && cpuctx->active_oncpu)
593 * Otherwise, try to add it if all previous groups were able
599 static void add_counter_to_ctx(struct perf_counter *counter,
600 struct perf_counter_context *ctx)
602 list_add_counter(counter, ctx);
603 counter->prev_state = PERF_COUNTER_STATE_OFF;
604 counter->tstamp_enabled = ctx->time;
605 counter->tstamp_running = ctx->time;
606 counter->tstamp_stopped = ctx->time;
610 * Cross CPU call to install and enable a performance counter
612 * Must be called with ctx->mutex held
614 static void __perf_install_in_context(void *info)
616 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
617 struct perf_counter *counter = info;
618 struct perf_counter_context *ctx = counter->ctx;
619 struct perf_counter *leader = counter->group_leader;
620 int cpu = smp_processor_id();
624 local_irq_save(flags);
626 * If this is a task context, we need to check whether it is
627 * the current task context of this cpu. If not it has been
628 * scheduled out before the smp call arrived.
629 * Or possibly this is the right context but it isn't
630 * on this cpu because it had no counters.
632 if (ctx->task && cpuctx->task_ctx != ctx) {
633 if (cpuctx->task_ctx || ctx->task != current) {
634 local_irq_restore(flags);
637 cpuctx->task_ctx = ctx;
640 spin_lock(&ctx->lock);
642 update_context_time(ctx);
645 * Protect the list operation against NMI by disabling the
646 * counters on a global level. NOP for non NMI based counters.
650 add_counter_to_ctx(counter, ctx);
653 * Don't put the counter on if it is disabled or if
654 * it is in a group and the group isn't on.
656 if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
657 (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
661 * An exclusive counter can't go on if there are already active
662 * hardware counters, and no hardware counter can go on if there
663 * is already an exclusive counter on.
665 if (!group_can_go_on(counter, cpuctx, 1))
668 err = counter_sched_in(counter, cpuctx, ctx, cpu);
672 * This counter couldn't go on. If it is in a group
673 * then we have to pull the whole group off.
674 * If the counter group is pinned then put it in error state.
676 if (leader != counter)
677 group_sched_out(leader, cpuctx, ctx);
678 if (leader->hw_event.pinned) {
679 update_group_times(leader);
680 leader->state = PERF_COUNTER_STATE_ERROR;
684 if (!err && !ctx->task && cpuctx->max_pertask)
685 cpuctx->max_pertask--;
690 spin_unlock_irqrestore(&ctx->lock, flags);
694 * Attach a performance counter to a context
696 * First we add the counter to the list with the hardware enable bit
697 * in counter->hw_config cleared.
699 * If the counter is attached to a task which is on a CPU we use a smp
700 * call to enable it in the task context. The task might have been
701 * scheduled away, but we check this in the smp call again.
703 * Must be called with ctx->mutex held.
706 perf_install_in_context(struct perf_counter_context *ctx,
707 struct perf_counter *counter,
710 struct task_struct *task = ctx->task;
714 * Per cpu counters are installed via an smp call and
715 * the install is always sucessful.
717 smp_call_function_single(cpu, __perf_install_in_context,
723 task_oncpu_function_call(task, __perf_install_in_context,
726 spin_lock_irq(&ctx->lock);
728 * we need to retry the smp call.
730 if (ctx->is_active && list_empty(&counter->list_entry)) {
731 spin_unlock_irq(&ctx->lock);
736 * The lock prevents that this context is scheduled in so we
737 * can add the counter safely, if it the call above did not
740 if (list_empty(&counter->list_entry))
741 add_counter_to_ctx(counter, ctx);
742 spin_unlock_irq(&ctx->lock);
746 * Cross CPU call to enable a performance counter
748 static void __perf_counter_enable(void *info)
750 struct perf_counter *counter = info;
751 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
752 struct perf_counter_context *ctx = counter->ctx;
753 struct perf_counter *leader = counter->group_leader;
757 local_irq_save(flags);
759 * If this is a per-task counter, need to check whether this
760 * counter's task is the current task on this cpu.
762 if (ctx->task && cpuctx->task_ctx != ctx) {
763 if (cpuctx->task_ctx || ctx->task != current) {
764 local_irq_restore(flags);
767 cpuctx->task_ctx = ctx;
770 spin_lock(&ctx->lock);
772 update_context_time(ctx);
774 counter->prev_state = counter->state;
775 if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
777 counter->state = PERF_COUNTER_STATE_INACTIVE;
778 counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
781 * If the counter is in a group and isn't the group leader,
782 * then don't put it on unless the group is on.
784 if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
787 if (!group_can_go_on(counter, cpuctx, 1)) {
791 if (counter == leader)
792 err = group_sched_in(counter, cpuctx, ctx,
795 err = counter_sched_in(counter, cpuctx, ctx,
802 * If this counter can't go on and it's part of a
803 * group, then the whole group has to come off.
805 if (leader != counter)
806 group_sched_out(leader, cpuctx, ctx);
807 if (leader->hw_event.pinned) {
808 update_group_times(leader);
809 leader->state = PERF_COUNTER_STATE_ERROR;
814 spin_unlock_irqrestore(&ctx->lock, flags);
820 * If counter->ctx is a cloned context, callers must make sure that
821 * every task struct that counter->ctx->task could possibly point to
822 * remains valid. This condition is satisfied when called through
823 * perf_counter_for_each_child or perf_counter_for_each as described
824 * for perf_counter_disable.
826 static void perf_counter_enable(struct perf_counter *counter)
828 struct perf_counter_context *ctx = counter->ctx;
829 struct task_struct *task = ctx->task;
833 * Enable the counter on the cpu that it's on
835 smp_call_function_single(counter->cpu, __perf_counter_enable,
840 spin_lock_irq(&ctx->lock);
841 if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
845 * If the counter is in error state, clear that first.
846 * That way, if we see the counter in error state below, we
847 * know that it has gone back into error state, as distinct
848 * from the task having been scheduled away before the
849 * cross-call arrived.
851 if (counter->state == PERF_COUNTER_STATE_ERROR)
852 counter->state = PERF_COUNTER_STATE_OFF;
855 spin_unlock_irq(&ctx->lock);
856 task_oncpu_function_call(task, __perf_counter_enable, counter);
858 spin_lock_irq(&ctx->lock);
861 * If the context is active and the counter is still off,
862 * we need to retry the cross-call.
864 if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
868 * Since we have the lock this context can't be scheduled
869 * in, so we can change the state safely.
871 if (counter->state == PERF_COUNTER_STATE_OFF) {
872 counter->state = PERF_COUNTER_STATE_INACTIVE;
873 counter->tstamp_enabled =
874 ctx->time - counter->total_time_enabled;
877 spin_unlock_irq(&ctx->lock);
880 static int perf_counter_refresh(struct perf_counter *counter, int refresh)
883 * not supported on inherited counters
885 if (counter->hw_event.inherit)
888 atomic_add(refresh, &counter->event_limit);
889 perf_counter_enable(counter);
894 void __perf_counter_sched_out(struct perf_counter_context *ctx,
895 struct perf_cpu_context *cpuctx)
897 struct perf_counter *counter;
899 spin_lock(&ctx->lock);
901 if (likely(!ctx->nr_counters))
903 update_context_time(ctx);
906 if (ctx->nr_active) {
907 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
908 if (counter != counter->group_leader)
909 counter_sched_out(counter, cpuctx, ctx);
911 group_sched_out(counter, cpuctx, ctx);
916 spin_unlock(&ctx->lock);
920 * Test whether two contexts are equivalent, i.e. whether they
921 * have both been cloned from the same version of the same context
922 * and they both have the same number of enabled counters.
923 * If the number of enabled counters is the same, then the set
924 * of enabled counters should be the same, because these are both
925 * inherited contexts, therefore we can't access individual counters
926 * in them directly with an fd; we can only enable/disable all
927 * counters via prctl, or enable/disable all counters in a family
928 * via ioctl, which will have the same effect on both contexts.
930 static int context_equiv(struct perf_counter_context *ctx1,
931 struct perf_counter_context *ctx2)
933 return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
934 && ctx1->parent_gen == ctx2->parent_gen
935 && ctx1->parent_gen != ~0ull;
939 * Called from scheduler to remove the counters of the current task,
940 * with interrupts disabled.
942 * We stop each counter and update the counter value in counter->count.
944 * This does not protect us against NMI, but disable()
945 * sets the disabled bit in the control field of counter _before_
946 * accessing the counter control register. If a NMI hits, then it will
947 * not restart the counter.
949 void perf_counter_task_sched_out(struct task_struct *task,
950 struct task_struct *next, int cpu)
952 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
953 struct perf_counter_context *ctx = task->perf_counter_ctxp;
954 struct perf_counter_context *next_ctx;
955 struct perf_counter_context *parent;
956 struct pt_regs *regs;
959 regs = task_pt_regs(task);
960 perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs, 0);
962 if (likely(!ctx || !cpuctx->task_ctx))
965 update_context_time(ctx);
968 parent = rcu_dereference(ctx->parent_ctx);
969 next_ctx = next->perf_counter_ctxp;
970 if (parent && next_ctx &&
971 rcu_dereference(next_ctx->parent_ctx) == parent) {
973 * Looks like the two contexts are clones, so we might be
974 * able to optimize the context switch. We lock both
975 * contexts and check that they are clones under the
976 * lock (including re-checking that neither has been
977 * uncloned in the meantime). It doesn't matter which
978 * order we take the locks because no other cpu could
979 * be trying to lock both of these tasks.
981 spin_lock(&ctx->lock);
982 spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
983 if (context_equiv(ctx, next_ctx)) {
984 task->perf_counter_ctxp = next_ctx;
985 next->perf_counter_ctxp = ctx;
987 next_ctx->task = task;
990 spin_unlock(&next_ctx->lock);
991 spin_unlock(&ctx->lock);
996 __perf_counter_sched_out(ctx, cpuctx);
997 cpuctx->task_ctx = NULL;
1001 static void __perf_counter_task_sched_out(struct perf_counter_context *ctx)
1003 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1005 if (!cpuctx->task_ctx)
1008 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
1011 __perf_counter_sched_out(ctx, cpuctx);
1012 cpuctx->task_ctx = NULL;
1015 static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
1017 __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
1021 __perf_counter_sched_in(struct perf_counter_context *ctx,
1022 struct perf_cpu_context *cpuctx, int cpu)
1024 struct perf_counter *counter;
1027 spin_lock(&ctx->lock);
1029 if (likely(!ctx->nr_counters))
1032 ctx->timestamp = perf_clock();
1037 * First go through the list and put on any pinned groups
1038 * in order to give them the best chance of going on.
1040 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1041 if (counter->state <= PERF_COUNTER_STATE_OFF ||
1042 !counter->hw_event.pinned)
1044 if (counter->cpu != -1 && counter->cpu != cpu)
1047 if (counter != counter->group_leader)
1048 counter_sched_in(counter, cpuctx, ctx, cpu);
1050 if (group_can_go_on(counter, cpuctx, 1))
1051 group_sched_in(counter, cpuctx, ctx, cpu);
1055 * If this pinned group hasn't been scheduled,
1056 * put it in error state.
1058 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1059 update_group_times(counter);
1060 counter->state = PERF_COUNTER_STATE_ERROR;
1064 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1066 * Ignore counters in OFF or ERROR state, and
1067 * ignore pinned counters since we did them already.
1069 if (counter->state <= PERF_COUNTER_STATE_OFF ||
1070 counter->hw_event.pinned)
1074 * Listen to the 'cpu' scheduling filter constraint
1077 if (counter->cpu != -1 && counter->cpu != cpu)
1080 if (counter != counter->group_leader) {
1081 if (counter_sched_in(counter, cpuctx, ctx, cpu))
1084 if (group_can_go_on(counter, cpuctx, can_add_hw)) {
1085 if (group_sched_in(counter, cpuctx, ctx, cpu))
1092 spin_unlock(&ctx->lock);
1096 * Called from scheduler to add the counters of the current task
1097 * with interrupts disabled.
1099 * We restore the counter value and then enable it.
1101 * This does not protect us against NMI, but enable()
1102 * sets the enabled bit in the control field of counter _before_
1103 * accessing the counter control register. If a NMI hits, then it will
1104 * keep the counter running.
1106 void perf_counter_task_sched_in(struct task_struct *task, int cpu)
1108 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1109 struct perf_counter_context *ctx = task->perf_counter_ctxp;
1113 if (cpuctx->task_ctx == ctx)
1115 __perf_counter_sched_in(ctx, cpuctx, cpu);
1116 cpuctx->task_ctx = ctx;
1119 static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
1121 struct perf_counter_context *ctx = &cpuctx->ctx;
1123 __perf_counter_sched_in(ctx, cpuctx, cpu);
1126 #define MAX_INTERRUPTS (~0ULL)
1128 static void perf_log_throttle(struct perf_counter *counter, int enable);
1129 static void perf_log_period(struct perf_counter *counter, u64 period);
1131 static void perf_adjust_freq(struct perf_counter_context *ctx)
1133 struct perf_counter *counter;
1134 u64 interrupts, irq_period;
1138 spin_lock(&ctx->lock);
1139 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1140 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
1143 interrupts = counter->hw.interrupts;
1144 counter->hw.interrupts = 0;
1146 if (interrupts == MAX_INTERRUPTS) {
1147 perf_log_throttle(counter, 1);
1148 counter->pmu->unthrottle(counter);
1149 interrupts = 2*sysctl_perf_counter_limit/HZ;
1152 if (!counter->hw_event.freq || !counter->hw_event.irq_freq)
1155 events = HZ * interrupts * counter->hw.irq_period;
1156 period = div64_u64(events, counter->hw_event.irq_freq);
1158 delta = (s64)(1 + period - counter->hw.irq_period);
1161 irq_period = counter->hw.irq_period + delta;
1166 perf_log_period(counter, irq_period);
1168 counter->hw.irq_period = irq_period;
1170 spin_unlock(&ctx->lock);
1174 * Round-robin a context's counters:
1176 static void rotate_ctx(struct perf_counter_context *ctx)
1178 struct perf_counter *counter;
1180 if (!ctx->nr_counters)
1183 spin_lock(&ctx->lock);
1185 * Rotate the first entry last (works just fine for group counters too):
1188 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1189 list_move_tail(&counter->list_entry, &ctx->counter_list);
1194 spin_unlock(&ctx->lock);
1197 void perf_counter_task_tick(struct task_struct *curr, int cpu)
1199 struct perf_cpu_context *cpuctx;
1200 struct perf_counter_context *ctx;
1202 if (!atomic_read(&nr_counters))
1205 cpuctx = &per_cpu(perf_cpu_context, cpu);
1206 ctx = curr->perf_counter_ctxp;
1208 perf_adjust_freq(&cpuctx->ctx);
1210 perf_adjust_freq(ctx);
1212 perf_counter_cpu_sched_out(cpuctx);
1214 __perf_counter_task_sched_out(ctx);
1216 rotate_ctx(&cpuctx->ctx);
1220 perf_counter_cpu_sched_in(cpuctx, cpu);
1222 perf_counter_task_sched_in(curr, cpu);
1226 * Cross CPU call to read the hardware counter
1228 static void __read(void *info)
1230 struct perf_counter *counter = info;
1231 struct perf_counter_context *ctx = counter->ctx;
1232 unsigned long flags;
1234 local_irq_save(flags);
1236 update_context_time(ctx);
1237 counter->pmu->read(counter);
1238 update_counter_times(counter);
1239 local_irq_restore(flags);
1242 static u64 perf_counter_read(struct perf_counter *counter)
1245 * If counter is enabled and currently active on a CPU, update the
1246 * value in the counter structure:
1248 if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
1249 smp_call_function_single(counter->oncpu,
1250 __read, counter, 1);
1251 } else if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1252 update_counter_times(counter);
1255 return atomic64_read(&counter->count);
1259 * Initialize the perf_counter context in a task_struct:
1262 __perf_counter_init_context(struct perf_counter_context *ctx,
1263 struct task_struct *task)
1265 memset(ctx, 0, sizeof(*ctx));
1266 spin_lock_init(&ctx->lock);
1267 mutex_init(&ctx->mutex);
1268 INIT_LIST_HEAD(&ctx->counter_list);
1269 INIT_LIST_HEAD(&ctx->event_list);
1270 atomic_set(&ctx->refcount, 1);
1274 static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1276 struct perf_cpu_context *cpuctx;
1277 struct perf_counter_context *ctx;
1278 struct perf_counter_context *parent_ctx;
1279 struct task_struct *task;
1283 * If cpu is not a wildcard then this is a percpu counter:
1286 /* Must be root to operate on a CPU counter: */
1287 if (sysctl_perf_counter_priv && !capable(CAP_SYS_ADMIN))
1288 return ERR_PTR(-EACCES);
1290 if (cpu < 0 || cpu > num_possible_cpus())
1291 return ERR_PTR(-EINVAL);
1294 * We could be clever and allow to attach a counter to an
1295 * offline CPU and activate it when the CPU comes up, but
1298 if (!cpu_isset(cpu, cpu_online_map))
1299 return ERR_PTR(-ENODEV);
1301 cpuctx = &per_cpu(perf_cpu_context, cpu);
1312 task = find_task_by_vpid(pid);
1314 get_task_struct(task);
1318 return ERR_PTR(-ESRCH);
1321 * Can't attach counters to a dying task.
1324 if (task->flags & PF_EXITING)
1327 /* Reuse ptrace permission checks for now. */
1329 if (!ptrace_may_access(task, PTRACE_MODE_READ))
1335 ctx = rcu_dereference(task->perf_counter_ctxp);
1338 * If this context is a clone of another, it might
1339 * get swapped for another underneath us by
1340 * perf_counter_task_sched_out, though the
1341 * rcu_read_lock() protects us from any context
1342 * getting freed. Lock the context and check if it
1343 * got swapped before we could get the lock, and retry
1344 * if so. If we locked the right context, then it
1345 * can't get swapped on us any more and we can
1346 * unclone it if necessary.
1347 * Once it's not a clone things will be stable.
1349 spin_lock_irq(&ctx->lock);
1350 if (ctx != rcu_dereference(task->perf_counter_ctxp)) {
1351 spin_unlock_irq(&ctx->lock);
1354 parent_ctx = ctx->parent_ctx;
1356 put_ctx(parent_ctx);
1357 ctx->parent_ctx = NULL; /* no longer a clone */
1360 * Get an extra reference before dropping the lock so that
1361 * this context won't get freed if the task exits.
1364 spin_unlock_irq(&ctx->lock);
1369 ctx = kmalloc(sizeof(struct perf_counter_context), GFP_KERNEL);
1373 __perf_counter_init_context(ctx, task);
1375 if (cmpxchg(&task->perf_counter_ctxp, NULL, ctx)) {
1377 * We raced with some other task; use
1378 * the context they set.
1383 get_task_struct(task);
1386 put_task_struct(task);
1390 put_task_struct(task);
1391 return ERR_PTR(err);
1394 static void free_counter_rcu(struct rcu_head *head)
1396 struct perf_counter *counter;
1398 counter = container_of(head, struct perf_counter, rcu_head);
1402 static void perf_pending_sync(struct perf_counter *counter);
1404 static void free_counter(struct perf_counter *counter)
1406 perf_pending_sync(counter);
1408 atomic_dec(&nr_counters);
1409 if (counter->hw_event.mmap)
1410 atomic_dec(&nr_mmap_tracking);
1411 if (counter->hw_event.munmap)
1412 atomic_dec(&nr_munmap_tracking);
1413 if (counter->hw_event.comm)
1414 atomic_dec(&nr_comm_tracking);
1416 if (counter->destroy)
1417 counter->destroy(counter);
1419 put_ctx(counter->ctx);
1420 call_rcu(&counter->rcu_head, free_counter_rcu);
1424 * Called when the last reference to the file is gone.
1426 static int perf_release(struct inode *inode, struct file *file)
1428 struct perf_counter *counter = file->private_data;
1429 struct perf_counter_context *ctx = counter->ctx;
1431 file->private_data = NULL;
1433 WARN_ON_ONCE(ctx->parent_ctx);
1434 mutex_lock(&ctx->mutex);
1435 perf_counter_remove_from_context(counter);
1436 mutex_unlock(&ctx->mutex);
1438 mutex_lock(&counter->owner->perf_counter_mutex);
1439 list_del_init(&counter->owner_entry);
1440 mutex_unlock(&counter->owner->perf_counter_mutex);
1441 put_task_struct(counter->owner);
1443 free_counter(counter);
1449 * Read the performance counter - simple non blocking version for now
1452 perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1458 * Return end-of-file for a read on a counter that is in
1459 * error state (i.e. because it was pinned but it couldn't be
1460 * scheduled on to the CPU at some point).
1462 if (counter->state == PERF_COUNTER_STATE_ERROR)
1465 WARN_ON_ONCE(counter->ctx->parent_ctx);
1466 mutex_lock(&counter->child_mutex);
1467 values[0] = perf_counter_read(counter);
1469 if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1470 values[n++] = counter->total_time_enabled +
1471 atomic64_read(&counter->child_total_time_enabled);
1472 if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1473 values[n++] = counter->total_time_running +
1474 atomic64_read(&counter->child_total_time_running);
1475 mutex_unlock(&counter->child_mutex);
1477 if (count < n * sizeof(u64))
1479 count = n * sizeof(u64);
1481 if (copy_to_user(buf, values, count))
1488 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1490 struct perf_counter *counter = file->private_data;
1492 return perf_read_hw(counter, buf, count);
1495 static unsigned int perf_poll(struct file *file, poll_table *wait)
1497 struct perf_counter *counter = file->private_data;
1498 struct perf_mmap_data *data;
1499 unsigned int events = POLL_HUP;
1502 data = rcu_dereference(counter->data);
1504 events = atomic_xchg(&data->poll, 0);
1507 poll_wait(file, &counter->waitq, wait);
1512 static void perf_counter_reset(struct perf_counter *counter)
1514 (void)perf_counter_read(counter);
1515 atomic64_set(&counter->count, 0);
1516 perf_counter_update_userpage(counter);
1519 static void perf_counter_for_each_sibling(struct perf_counter *counter,
1520 void (*func)(struct perf_counter *))
1522 struct perf_counter_context *ctx = counter->ctx;
1523 struct perf_counter *sibling;
1525 WARN_ON_ONCE(ctx->parent_ctx);
1526 mutex_lock(&ctx->mutex);
1527 counter = counter->group_leader;
1530 list_for_each_entry(sibling, &counter->sibling_list, list_entry)
1532 mutex_unlock(&ctx->mutex);
1536 * Holding the top-level counter's child_mutex means that any
1537 * descendant process that has inherited this counter will block
1538 * in sync_child_counter if it goes to exit, thus satisfying the
1539 * task existence requirements of perf_counter_enable/disable.
1541 static void perf_counter_for_each_child(struct perf_counter *counter,
1542 void (*func)(struct perf_counter *))
1544 struct perf_counter *child;
1546 WARN_ON_ONCE(counter->ctx->parent_ctx);
1547 mutex_lock(&counter->child_mutex);
1549 list_for_each_entry(child, &counter->child_list, child_list)
1551 mutex_unlock(&counter->child_mutex);
1554 static void perf_counter_for_each(struct perf_counter *counter,
1555 void (*func)(struct perf_counter *))
1557 struct perf_counter *child;
1559 WARN_ON_ONCE(counter->ctx->parent_ctx);
1560 mutex_lock(&counter->child_mutex);
1561 perf_counter_for_each_sibling(counter, func);
1562 list_for_each_entry(child, &counter->child_list, child_list)
1563 perf_counter_for_each_sibling(child, func);
1564 mutex_unlock(&counter->child_mutex);
1567 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1569 struct perf_counter *counter = file->private_data;
1570 void (*func)(struct perf_counter *);
1574 case PERF_COUNTER_IOC_ENABLE:
1575 func = perf_counter_enable;
1577 case PERF_COUNTER_IOC_DISABLE:
1578 func = perf_counter_disable;
1580 case PERF_COUNTER_IOC_RESET:
1581 func = perf_counter_reset;
1584 case PERF_COUNTER_IOC_REFRESH:
1585 return perf_counter_refresh(counter, arg);
1590 if (flags & PERF_IOC_FLAG_GROUP)
1591 perf_counter_for_each(counter, func);
1593 perf_counter_for_each_child(counter, func);
1598 int perf_counter_task_enable(void)
1600 struct perf_counter *counter;
1602 mutex_lock(¤t->perf_counter_mutex);
1603 list_for_each_entry(counter, ¤t->perf_counter_list, owner_entry)
1604 perf_counter_for_each_child(counter, perf_counter_enable);
1605 mutex_unlock(¤t->perf_counter_mutex);
1610 int perf_counter_task_disable(void)
1612 struct perf_counter *counter;
1614 mutex_lock(¤t->perf_counter_mutex);
1615 list_for_each_entry(counter, ¤t->perf_counter_list, owner_entry)
1616 perf_counter_for_each_child(counter, perf_counter_disable);
1617 mutex_unlock(¤t->perf_counter_mutex);
1623 * Callers need to ensure there can be no nesting of this function, otherwise
1624 * the seqlock logic goes bad. We can not serialize this because the arch
1625 * code calls this from NMI context.
1627 void perf_counter_update_userpage(struct perf_counter *counter)
1629 struct perf_mmap_data *data;
1630 struct perf_counter_mmap_page *userpg;
1633 data = rcu_dereference(counter->data);
1637 userpg = data->user_page;
1640 * Disable preemption so as to not let the corresponding user-space
1641 * spin too long if we get preempted.
1646 userpg->index = counter->hw.idx;
1647 userpg->offset = atomic64_read(&counter->count);
1648 if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1649 userpg->offset -= atomic64_read(&counter->hw.prev_count);
1658 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1660 struct perf_counter *counter = vma->vm_file->private_data;
1661 struct perf_mmap_data *data;
1662 int ret = VM_FAULT_SIGBUS;
1665 data = rcu_dereference(counter->data);
1669 if (vmf->pgoff == 0) {
1670 vmf->page = virt_to_page(data->user_page);
1672 int nr = vmf->pgoff - 1;
1674 if ((unsigned)nr > data->nr_pages)
1677 vmf->page = virt_to_page(data->data_pages[nr]);
1679 get_page(vmf->page);
1687 static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
1689 struct perf_mmap_data *data;
1693 WARN_ON(atomic_read(&counter->mmap_count));
1695 size = sizeof(struct perf_mmap_data);
1696 size += nr_pages * sizeof(void *);
1698 data = kzalloc(size, GFP_KERNEL);
1702 data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
1703 if (!data->user_page)
1704 goto fail_user_page;
1706 for (i = 0; i < nr_pages; i++) {
1707 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
1708 if (!data->data_pages[i])
1709 goto fail_data_pages;
1712 data->nr_pages = nr_pages;
1713 atomic_set(&data->lock, -1);
1715 rcu_assign_pointer(counter->data, data);
1720 for (i--; i >= 0; i--)
1721 free_page((unsigned long)data->data_pages[i]);
1723 free_page((unsigned long)data->user_page);
1732 static void __perf_mmap_data_free(struct rcu_head *rcu_head)
1734 struct perf_mmap_data *data = container_of(rcu_head,
1735 struct perf_mmap_data, rcu_head);
1738 free_page((unsigned long)data->user_page);
1739 for (i = 0; i < data->nr_pages; i++)
1740 free_page((unsigned long)data->data_pages[i]);
1744 static void perf_mmap_data_free(struct perf_counter *counter)
1746 struct perf_mmap_data *data = counter->data;
1748 WARN_ON(atomic_read(&counter->mmap_count));
1750 rcu_assign_pointer(counter->data, NULL);
1751 call_rcu(&data->rcu_head, __perf_mmap_data_free);
1754 static void perf_mmap_open(struct vm_area_struct *vma)
1756 struct perf_counter *counter = vma->vm_file->private_data;
1758 atomic_inc(&counter->mmap_count);
1761 static void perf_mmap_close(struct vm_area_struct *vma)
1763 struct perf_counter *counter = vma->vm_file->private_data;
1765 WARN_ON_ONCE(counter->ctx->parent_ctx);
1766 if (atomic_dec_and_mutex_lock(&counter->mmap_count,
1767 &counter->mmap_mutex)) {
1768 struct user_struct *user = current_user();
1770 atomic_long_sub(counter->data->nr_pages + 1, &user->locked_vm);
1771 vma->vm_mm->locked_vm -= counter->data->nr_locked;
1772 perf_mmap_data_free(counter);
1773 mutex_unlock(&counter->mmap_mutex);
1777 static struct vm_operations_struct perf_mmap_vmops = {
1778 .open = perf_mmap_open,
1779 .close = perf_mmap_close,
1780 .fault = perf_mmap_fault,
1783 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
1785 struct perf_counter *counter = file->private_data;
1786 struct user_struct *user = current_user();
1787 unsigned long vma_size;
1788 unsigned long nr_pages;
1789 unsigned long user_locked, user_lock_limit;
1790 unsigned long locked, lock_limit;
1791 long user_extra, extra;
1794 if (!(vma->vm_flags & VM_SHARED) || (vma->vm_flags & VM_WRITE))
1797 vma_size = vma->vm_end - vma->vm_start;
1798 nr_pages = (vma_size / PAGE_SIZE) - 1;
1801 * If we have data pages ensure they're a power-of-two number, so we
1802 * can do bitmasks instead of modulo.
1804 if (nr_pages != 0 && !is_power_of_2(nr_pages))
1807 if (vma_size != PAGE_SIZE * (1 + nr_pages))
1810 if (vma->vm_pgoff != 0)
1813 WARN_ON_ONCE(counter->ctx->parent_ctx);
1814 mutex_lock(&counter->mmap_mutex);
1815 if (atomic_inc_not_zero(&counter->mmap_count)) {
1816 if (nr_pages != counter->data->nr_pages)
1821 user_extra = nr_pages + 1;
1822 user_lock_limit = sysctl_perf_counter_mlock >> (PAGE_SHIFT - 10);
1825 * Increase the limit linearly with more CPUs:
1827 user_lock_limit *= num_online_cpus();
1829 user_locked = atomic_long_read(&user->locked_vm) + user_extra;
1832 if (user_locked > user_lock_limit)
1833 extra = user_locked - user_lock_limit;
1835 lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
1836 lock_limit >>= PAGE_SHIFT;
1837 locked = vma->vm_mm->locked_vm + extra;
1839 if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
1844 WARN_ON(counter->data);
1845 ret = perf_mmap_data_alloc(counter, nr_pages);
1849 atomic_set(&counter->mmap_count, 1);
1850 atomic_long_add(user_extra, &user->locked_vm);
1851 vma->vm_mm->locked_vm += extra;
1852 counter->data->nr_locked = extra;
1854 mutex_unlock(&counter->mmap_mutex);
1856 vma->vm_flags &= ~VM_MAYWRITE;
1857 vma->vm_flags |= VM_RESERVED;
1858 vma->vm_ops = &perf_mmap_vmops;
1863 static int perf_fasync(int fd, struct file *filp, int on)
1865 struct perf_counter *counter = filp->private_data;
1866 struct inode *inode = filp->f_path.dentry->d_inode;
1869 mutex_lock(&inode->i_mutex);
1870 retval = fasync_helper(fd, filp, on, &counter->fasync);
1871 mutex_unlock(&inode->i_mutex);
1879 static const struct file_operations perf_fops = {
1880 .release = perf_release,
1883 .unlocked_ioctl = perf_ioctl,
1884 .compat_ioctl = perf_ioctl,
1886 .fasync = perf_fasync,
1890 * Perf counter wakeup
1892 * If there's data, ensure we set the poll() state and publish everything
1893 * to user-space before waking everybody up.
1896 void perf_counter_wakeup(struct perf_counter *counter)
1898 wake_up_all(&counter->waitq);
1900 if (counter->pending_kill) {
1901 kill_fasync(&counter->fasync, SIGIO, counter->pending_kill);
1902 counter->pending_kill = 0;
1909 * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
1911 * The NMI bit means we cannot possibly take locks. Therefore, maintain a
1912 * single linked list and use cmpxchg() to add entries lockless.
1915 static void perf_pending_counter(struct perf_pending_entry *entry)
1917 struct perf_counter *counter = container_of(entry,
1918 struct perf_counter, pending);
1920 if (counter->pending_disable) {
1921 counter->pending_disable = 0;
1922 perf_counter_disable(counter);
1925 if (counter->pending_wakeup) {
1926 counter->pending_wakeup = 0;
1927 perf_counter_wakeup(counter);
1931 #define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
1933 static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
1937 static void perf_pending_queue(struct perf_pending_entry *entry,
1938 void (*func)(struct perf_pending_entry *))
1940 struct perf_pending_entry **head;
1942 if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
1947 head = &get_cpu_var(perf_pending_head);
1950 entry->next = *head;
1951 } while (cmpxchg(head, entry->next, entry) != entry->next);
1953 set_perf_counter_pending();
1955 put_cpu_var(perf_pending_head);
1958 static int __perf_pending_run(void)
1960 struct perf_pending_entry *list;
1963 list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
1964 while (list != PENDING_TAIL) {
1965 void (*func)(struct perf_pending_entry *);
1966 struct perf_pending_entry *entry = list;
1973 * Ensure we observe the unqueue before we issue the wakeup,
1974 * so that we won't be waiting forever.
1975 * -- see perf_not_pending().
1986 static inline int perf_not_pending(struct perf_counter *counter)
1989 * If we flush on whatever cpu we run, there is a chance we don't
1993 __perf_pending_run();
1997 * Ensure we see the proper queue state before going to sleep
1998 * so that we do not miss the wakeup. -- see perf_pending_handle()
2001 return counter->pending.next == NULL;
2004 static void perf_pending_sync(struct perf_counter *counter)
2006 wait_event(counter->waitq, perf_not_pending(counter));
2009 void perf_counter_do_pending(void)
2011 __perf_pending_run();
2015 * Callchain support -- arch specific
2018 __weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2027 struct perf_output_handle {
2028 struct perf_counter *counter;
2029 struct perf_mmap_data *data;
2030 unsigned int offset;
2035 unsigned long flags;
2038 static void perf_output_wakeup(struct perf_output_handle *handle)
2040 atomic_set(&handle->data->poll, POLL_IN);
2043 handle->counter->pending_wakeup = 1;
2044 perf_pending_queue(&handle->counter->pending,
2045 perf_pending_counter);
2047 perf_counter_wakeup(handle->counter);
2051 * Curious locking construct.
2053 * We need to ensure a later event doesn't publish a head when a former
2054 * event isn't done writing. However since we need to deal with NMIs we
2055 * cannot fully serialize things.
2057 * What we do is serialize between CPUs so we only have to deal with NMI
2058 * nesting on a single CPU.
2060 * We only publish the head (and generate a wakeup) when the outer-most
2063 static void perf_output_lock(struct perf_output_handle *handle)
2065 struct perf_mmap_data *data = handle->data;
2070 local_irq_save(handle->flags);
2071 cpu = smp_processor_id();
2073 if (in_nmi() && atomic_read(&data->lock) == cpu)
2076 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2082 static void perf_output_unlock(struct perf_output_handle *handle)
2084 struct perf_mmap_data *data = handle->data;
2087 data->done_head = data->head;
2089 if (!handle->locked)
2094 * The xchg implies a full barrier that ensures all writes are done
2095 * before we publish the new head, matched by a rmb() in userspace when
2096 * reading this position.
2098 while ((head = atomic_xchg(&data->done_head, 0)))
2099 data->user_page->data_head = head;
2102 * NMI can happen here, which means we can miss a done_head update.
2105 cpu = atomic_xchg(&data->lock, -1);
2106 WARN_ON_ONCE(cpu != smp_processor_id());
2109 * Therefore we have to validate we did not indeed do so.
2111 if (unlikely(atomic_read(&data->done_head))) {
2113 * Since we had it locked, we can lock it again.
2115 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2121 if (atomic_xchg(&data->wakeup, 0))
2122 perf_output_wakeup(handle);
2124 local_irq_restore(handle->flags);
2127 static int perf_output_begin(struct perf_output_handle *handle,
2128 struct perf_counter *counter, unsigned int size,
2129 int nmi, int overflow)
2131 struct perf_mmap_data *data;
2132 unsigned int offset, head;
2135 * For inherited counters we send all the output towards the parent.
2137 if (counter->parent)
2138 counter = counter->parent;
2141 data = rcu_dereference(counter->data);
2145 handle->data = data;
2146 handle->counter = counter;
2148 handle->overflow = overflow;
2150 if (!data->nr_pages)
2153 perf_output_lock(handle);
2156 offset = head = atomic_read(&data->head);
2158 } while (atomic_cmpxchg(&data->head, offset, head) != offset);
2160 handle->offset = offset;
2161 handle->head = head;
2163 if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
2164 atomic_set(&data->wakeup, 1);
2169 perf_output_wakeup(handle);
2176 static void perf_output_copy(struct perf_output_handle *handle,
2177 void *buf, unsigned int len)
2179 unsigned int pages_mask;
2180 unsigned int offset;
2184 offset = handle->offset;
2185 pages_mask = handle->data->nr_pages - 1;
2186 pages = handle->data->data_pages;
2189 unsigned int page_offset;
2192 nr = (offset >> PAGE_SHIFT) & pages_mask;
2193 page_offset = offset & (PAGE_SIZE - 1);
2194 size = min_t(unsigned int, PAGE_SIZE - page_offset, len);
2196 memcpy(pages[nr] + page_offset, buf, size);
2203 handle->offset = offset;
2206 * Check we didn't copy past our reservation window, taking the
2207 * possible unsigned int wrap into account.
2209 WARN_ON_ONCE(((int)(handle->head - handle->offset)) < 0);
2212 #define perf_output_put(handle, x) \
2213 perf_output_copy((handle), &(x), sizeof(x))
2215 static void perf_output_end(struct perf_output_handle *handle)
2217 struct perf_counter *counter = handle->counter;
2218 struct perf_mmap_data *data = handle->data;
2220 int wakeup_events = counter->hw_event.wakeup_events;
2222 if (handle->overflow && wakeup_events) {
2223 int events = atomic_inc_return(&data->events);
2224 if (events >= wakeup_events) {
2225 atomic_sub(wakeup_events, &data->events);
2226 atomic_set(&data->wakeup, 1);
2230 perf_output_unlock(handle);
2234 static void perf_counter_output(struct perf_counter *counter,
2235 int nmi, struct pt_regs *regs, u64 addr)
2238 u64 record_type = counter->hw_event.record_type;
2239 struct perf_output_handle handle;
2240 struct perf_event_header header;
2249 struct perf_callchain_entry *callchain = NULL;
2250 int callchain_size = 0;
2257 header.size = sizeof(header);
2259 header.misc = PERF_EVENT_MISC_OVERFLOW;
2260 header.misc |= perf_misc_flags(regs);
2262 if (record_type & PERF_RECORD_IP) {
2263 ip = perf_instruction_pointer(regs);
2264 header.type |= PERF_RECORD_IP;
2265 header.size += sizeof(ip);
2268 if (record_type & PERF_RECORD_TID) {
2269 /* namespace issues */
2270 tid_entry.pid = current->group_leader->pid;
2271 tid_entry.tid = current->pid;
2273 header.type |= PERF_RECORD_TID;
2274 header.size += sizeof(tid_entry);
2277 if (record_type & PERF_RECORD_TIME) {
2279 * Maybe do better on x86 and provide cpu_clock_nmi()
2281 time = sched_clock();
2283 header.type |= PERF_RECORD_TIME;
2284 header.size += sizeof(u64);
2287 if (record_type & PERF_RECORD_ADDR) {
2288 header.type |= PERF_RECORD_ADDR;
2289 header.size += sizeof(u64);
2292 if (record_type & PERF_RECORD_CONFIG) {
2293 header.type |= PERF_RECORD_CONFIG;
2294 header.size += sizeof(u64);
2297 if (record_type & PERF_RECORD_CPU) {
2298 header.type |= PERF_RECORD_CPU;
2299 header.size += sizeof(cpu_entry);
2301 cpu_entry.cpu = raw_smp_processor_id();
2304 if (record_type & PERF_RECORD_GROUP) {
2305 header.type |= PERF_RECORD_GROUP;
2306 header.size += sizeof(u64) +
2307 counter->nr_siblings * sizeof(group_entry);
2310 if (record_type & PERF_RECORD_CALLCHAIN) {
2311 callchain = perf_callchain(regs);
2314 callchain_size = (1 + callchain->nr) * sizeof(u64);
2316 header.type |= PERF_RECORD_CALLCHAIN;
2317 header.size += callchain_size;
2321 ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
2325 perf_output_put(&handle, header);
2327 if (record_type & PERF_RECORD_IP)
2328 perf_output_put(&handle, ip);
2330 if (record_type & PERF_RECORD_TID)
2331 perf_output_put(&handle, tid_entry);
2333 if (record_type & PERF_RECORD_TIME)
2334 perf_output_put(&handle, time);
2336 if (record_type & PERF_RECORD_ADDR)
2337 perf_output_put(&handle, addr);
2339 if (record_type & PERF_RECORD_CONFIG)
2340 perf_output_put(&handle, counter->hw_event.config);
2342 if (record_type & PERF_RECORD_CPU)
2343 perf_output_put(&handle, cpu_entry);
2346 * XXX PERF_RECORD_GROUP vs inherited counters seems difficult.
2348 if (record_type & PERF_RECORD_GROUP) {
2349 struct perf_counter *leader, *sub;
2350 u64 nr = counter->nr_siblings;
2352 perf_output_put(&handle, nr);
2354 leader = counter->group_leader;
2355 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
2357 sub->pmu->read(sub);
2359 group_entry.event = sub->hw_event.config;
2360 group_entry.counter = atomic64_read(&sub->count);
2362 perf_output_put(&handle, group_entry);
2367 perf_output_copy(&handle, callchain, callchain_size);
2369 perf_output_end(&handle);
2376 struct perf_comm_event {
2377 struct task_struct *task;
2382 struct perf_event_header header;
2389 static void perf_counter_comm_output(struct perf_counter *counter,
2390 struct perf_comm_event *comm_event)
2392 struct perf_output_handle handle;
2393 int size = comm_event->event.header.size;
2394 int ret = perf_output_begin(&handle, counter, size, 0, 0);
2399 perf_output_put(&handle, comm_event->event);
2400 perf_output_copy(&handle, comm_event->comm,
2401 comm_event->comm_size);
2402 perf_output_end(&handle);
2405 static int perf_counter_comm_match(struct perf_counter *counter,
2406 struct perf_comm_event *comm_event)
2408 if (counter->hw_event.comm &&
2409 comm_event->event.header.type == PERF_EVENT_COMM)
2415 static void perf_counter_comm_ctx(struct perf_counter_context *ctx,
2416 struct perf_comm_event *comm_event)
2418 struct perf_counter *counter;
2420 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2424 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2425 if (perf_counter_comm_match(counter, comm_event))
2426 perf_counter_comm_output(counter, comm_event);
2431 static void perf_counter_comm_event(struct perf_comm_event *comm_event)
2433 struct perf_cpu_context *cpuctx;
2435 char *comm = comm_event->task->comm;
2437 size = ALIGN(strlen(comm)+1, sizeof(u64));
2439 comm_event->comm = comm;
2440 comm_event->comm_size = size;
2442 comm_event->event.header.size = sizeof(comm_event->event) + size;
2444 cpuctx = &get_cpu_var(perf_cpu_context);
2445 perf_counter_comm_ctx(&cpuctx->ctx, comm_event);
2446 put_cpu_var(perf_cpu_context);
2448 perf_counter_comm_ctx(current->perf_counter_ctxp, comm_event);
2451 void perf_counter_comm(struct task_struct *task)
2453 struct perf_comm_event comm_event;
2455 if (!atomic_read(&nr_comm_tracking))
2457 if (!current->perf_counter_ctxp)
2460 comm_event = (struct perf_comm_event){
2463 .header = { .type = PERF_EVENT_COMM, },
2464 .pid = task->group_leader->pid,
2469 perf_counter_comm_event(&comm_event);
2476 struct perf_mmap_event {
2482 struct perf_event_header header;
2492 static void perf_counter_mmap_output(struct perf_counter *counter,
2493 struct perf_mmap_event *mmap_event)
2495 struct perf_output_handle handle;
2496 int size = mmap_event->event.header.size;
2497 int ret = perf_output_begin(&handle, counter, size, 0, 0);
2502 perf_output_put(&handle, mmap_event->event);
2503 perf_output_copy(&handle, mmap_event->file_name,
2504 mmap_event->file_size);
2505 perf_output_end(&handle);
2508 static int perf_counter_mmap_match(struct perf_counter *counter,
2509 struct perf_mmap_event *mmap_event)
2511 if (counter->hw_event.mmap &&
2512 mmap_event->event.header.type == PERF_EVENT_MMAP)
2515 if (counter->hw_event.munmap &&
2516 mmap_event->event.header.type == PERF_EVENT_MUNMAP)
2522 static void perf_counter_mmap_ctx(struct perf_counter_context *ctx,
2523 struct perf_mmap_event *mmap_event)
2525 struct perf_counter *counter;
2527 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2531 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2532 if (perf_counter_mmap_match(counter, mmap_event))
2533 perf_counter_mmap_output(counter, mmap_event);
2538 static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
2540 struct perf_cpu_context *cpuctx;
2541 struct file *file = mmap_event->file;
2548 buf = kzalloc(PATH_MAX, GFP_KERNEL);
2550 name = strncpy(tmp, "//enomem", sizeof(tmp));
2553 name = d_path(&file->f_path, buf, PATH_MAX);
2555 name = strncpy(tmp, "//toolong", sizeof(tmp));
2559 name = strncpy(tmp, "//anon", sizeof(tmp));
2564 size = ALIGN(strlen(name)+1, sizeof(u64));
2566 mmap_event->file_name = name;
2567 mmap_event->file_size = size;
2569 mmap_event->event.header.size = sizeof(mmap_event->event) + size;
2571 cpuctx = &get_cpu_var(perf_cpu_context);
2572 perf_counter_mmap_ctx(&cpuctx->ctx, mmap_event);
2573 put_cpu_var(perf_cpu_context);
2575 perf_counter_mmap_ctx(current->perf_counter_ctxp, mmap_event);
2580 void perf_counter_mmap(unsigned long addr, unsigned long len,
2581 unsigned long pgoff, struct file *file)
2583 struct perf_mmap_event mmap_event;
2585 if (!atomic_read(&nr_mmap_tracking))
2587 if (!current->perf_counter_ctxp)
2590 mmap_event = (struct perf_mmap_event){
2593 .header = { .type = PERF_EVENT_MMAP, },
2594 .pid = current->group_leader->pid,
2595 .tid = current->pid,
2602 perf_counter_mmap_event(&mmap_event);
2605 void perf_counter_munmap(unsigned long addr, unsigned long len,
2606 unsigned long pgoff, struct file *file)
2608 struct perf_mmap_event mmap_event;
2610 if (!atomic_read(&nr_munmap_tracking))
2613 mmap_event = (struct perf_mmap_event){
2616 .header = { .type = PERF_EVENT_MUNMAP, },
2617 .pid = current->group_leader->pid,
2618 .tid = current->pid,
2625 perf_counter_mmap_event(&mmap_event);
2629 * Log irq_period changes so that analyzing tools can re-normalize the
2633 static void perf_log_period(struct perf_counter *counter, u64 period)
2635 struct perf_output_handle handle;
2639 struct perf_event_header header;
2644 .type = PERF_EVENT_PERIOD,
2646 .size = sizeof(freq_event),
2648 .time = sched_clock(),
2652 if (counter->hw.irq_period == period)
2655 ret = perf_output_begin(&handle, counter, sizeof(freq_event), 0, 0);
2659 perf_output_put(&handle, freq_event);
2660 perf_output_end(&handle);
2664 * IRQ throttle logging
2667 static void perf_log_throttle(struct perf_counter *counter, int enable)
2669 struct perf_output_handle handle;
2673 struct perf_event_header header;
2675 } throttle_event = {
2677 .type = PERF_EVENT_THROTTLE + 1,
2679 .size = sizeof(throttle_event),
2681 .time = sched_clock(),
2684 ret = perf_output_begin(&handle, counter, sizeof(throttle_event), 1, 0);
2688 perf_output_put(&handle, throttle_event);
2689 perf_output_end(&handle);
2693 * Generic counter overflow handling.
2696 int perf_counter_overflow(struct perf_counter *counter,
2697 int nmi, struct pt_regs *regs, u64 addr)
2699 int events = atomic_read(&counter->event_limit);
2700 int throttle = counter->pmu->unthrottle != NULL;
2704 counter->hw.interrupts++;
2705 } else if (counter->hw.interrupts != MAX_INTERRUPTS) {
2706 counter->hw.interrupts++;
2707 if (HZ*counter->hw.interrupts > (u64)sysctl_perf_counter_limit) {
2708 counter->hw.interrupts = MAX_INTERRUPTS;
2709 perf_log_throttle(counter, 0);
2715 * XXX event_limit might not quite work as expected on inherited
2719 counter->pending_kill = POLL_IN;
2720 if (events && atomic_dec_and_test(&counter->event_limit)) {
2722 counter->pending_kill = POLL_HUP;
2724 counter->pending_disable = 1;
2725 perf_pending_queue(&counter->pending,
2726 perf_pending_counter);
2728 perf_counter_disable(counter);
2731 perf_counter_output(counter, nmi, regs, addr);
2736 * Generic software counter infrastructure
2739 static void perf_swcounter_update(struct perf_counter *counter)
2741 struct hw_perf_counter *hwc = &counter->hw;
2746 prev = atomic64_read(&hwc->prev_count);
2747 now = atomic64_read(&hwc->count);
2748 if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev)
2753 atomic64_add(delta, &counter->count);
2754 atomic64_sub(delta, &hwc->period_left);
2757 static void perf_swcounter_set_period(struct perf_counter *counter)
2759 struct hw_perf_counter *hwc = &counter->hw;
2760 s64 left = atomic64_read(&hwc->period_left);
2761 s64 period = hwc->irq_period;
2763 if (unlikely(left <= -period)) {
2765 atomic64_set(&hwc->period_left, left);
2768 if (unlikely(left <= 0)) {
2770 atomic64_add(period, &hwc->period_left);
2773 atomic64_set(&hwc->prev_count, -left);
2774 atomic64_set(&hwc->count, -left);
2777 static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
2779 enum hrtimer_restart ret = HRTIMER_RESTART;
2780 struct perf_counter *counter;
2781 struct pt_regs *regs;
2784 counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
2785 counter->pmu->read(counter);
2787 regs = get_irq_regs();
2789 * In case we exclude kernel IPs or are somehow not in interrupt
2790 * context, provide the next best thing, the user IP.
2792 if ((counter->hw_event.exclude_kernel || !regs) &&
2793 !counter->hw_event.exclude_user)
2794 regs = task_pt_regs(current);
2797 if (perf_counter_overflow(counter, 0, regs, 0))
2798 ret = HRTIMER_NORESTART;
2801 period = max_t(u64, 10000, counter->hw.irq_period);
2802 hrtimer_forward_now(hrtimer, ns_to_ktime(period));
2807 static void perf_swcounter_overflow(struct perf_counter *counter,
2808 int nmi, struct pt_regs *regs, u64 addr)
2810 perf_swcounter_update(counter);
2811 perf_swcounter_set_period(counter);
2812 if (perf_counter_overflow(counter, nmi, regs, addr))
2813 /* soft-disable the counter */
2818 static int perf_swcounter_match(struct perf_counter *counter,
2819 enum perf_event_types type,
2820 u32 event, struct pt_regs *regs)
2822 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
2825 if (perf_event_raw(&counter->hw_event))
2828 if (perf_event_type(&counter->hw_event) != type)
2831 if (perf_event_id(&counter->hw_event) != event)
2834 if (counter->hw_event.exclude_user && user_mode(regs))
2837 if (counter->hw_event.exclude_kernel && !user_mode(regs))
2843 static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
2844 int nmi, struct pt_regs *regs, u64 addr)
2846 int neg = atomic64_add_negative(nr, &counter->hw.count);
2847 if (counter->hw.irq_period && !neg)
2848 perf_swcounter_overflow(counter, nmi, regs, addr);
2851 static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
2852 enum perf_event_types type, u32 event,
2853 u64 nr, int nmi, struct pt_regs *regs,
2856 struct perf_counter *counter;
2858 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2862 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2863 if (perf_swcounter_match(counter, type, event, regs))
2864 perf_swcounter_add(counter, nr, nmi, regs, addr);
2869 static int *perf_swcounter_recursion_context(struct perf_cpu_context *cpuctx)
2872 return &cpuctx->recursion[3];
2875 return &cpuctx->recursion[2];
2878 return &cpuctx->recursion[1];
2880 return &cpuctx->recursion[0];
2883 static void __perf_swcounter_event(enum perf_event_types type, u32 event,
2884 u64 nr, int nmi, struct pt_regs *regs,
2887 struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
2888 int *recursion = perf_swcounter_recursion_context(cpuctx);
2896 perf_swcounter_ctx_event(&cpuctx->ctx, type, event,
2897 nr, nmi, regs, addr);
2898 if (cpuctx->task_ctx) {
2899 perf_swcounter_ctx_event(cpuctx->task_ctx, type, event,
2900 nr, nmi, regs, addr);
2907 put_cpu_var(perf_cpu_context);
2911 perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
2913 __perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, regs, addr);
2916 static void perf_swcounter_read(struct perf_counter *counter)
2918 perf_swcounter_update(counter);
2921 static int perf_swcounter_enable(struct perf_counter *counter)
2923 perf_swcounter_set_period(counter);
2927 static void perf_swcounter_disable(struct perf_counter *counter)
2929 perf_swcounter_update(counter);
2932 static const struct pmu perf_ops_generic = {
2933 .enable = perf_swcounter_enable,
2934 .disable = perf_swcounter_disable,
2935 .read = perf_swcounter_read,
2939 * Software counter: cpu wall time clock
2942 static void cpu_clock_perf_counter_update(struct perf_counter *counter)
2944 int cpu = raw_smp_processor_id();
2948 now = cpu_clock(cpu);
2949 prev = atomic64_read(&counter->hw.prev_count);
2950 atomic64_set(&counter->hw.prev_count, now);
2951 atomic64_add(now - prev, &counter->count);
2954 static int cpu_clock_perf_counter_enable(struct perf_counter *counter)
2956 struct hw_perf_counter *hwc = &counter->hw;
2957 int cpu = raw_smp_processor_id();
2959 atomic64_set(&hwc->prev_count, cpu_clock(cpu));
2960 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2961 hwc->hrtimer.function = perf_swcounter_hrtimer;
2962 if (hwc->irq_period) {
2963 u64 period = max_t(u64, 10000, hwc->irq_period);
2964 __hrtimer_start_range_ns(&hwc->hrtimer,
2965 ns_to_ktime(period), 0,
2966 HRTIMER_MODE_REL, 0);
2972 static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
2974 if (counter->hw.irq_period)
2975 hrtimer_cancel(&counter->hw.hrtimer);
2976 cpu_clock_perf_counter_update(counter);
2979 static void cpu_clock_perf_counter_read(struct perf_counter *counter)
2981 cpu_clock_perf_counter_update(counter);
2984 static const struct pmu perf_ops_cpu_clock = {
2985 .enable = cpu_clock_perf_counter_enable,
2986 .disable = cpu_clock_perf_counter_disable,
2987 .read = cpu_clock_perf_counter_read,
2991 * Software counter: task time clock
2994 static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
2999 prev = atomic64_xchg(&counter->hw.prev_count, now);
3001 atomic64_add(delta, &counter->count);
3004 static int task_clock_perf_counter_enable(struct perf_counter *counter)
3006 struct hw_perf_counter *hwc = &counter->hw;
3009 now = counter->ctx->time;
3011 atomic64_set(&hwc->prev_count, now);
3012 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3013 hwc->hrtimer.function = perf_swcounter_hrtimer;
3014 if (hwc->irq_period) {
3015 u64 period = max_t(u64, 10000, hwc->irq_period);
3016 __hrtimer_start_range_ns(&hwc->hrtimer,
3017 ns_to_ktime(period), 0,
3018 HRTIMER_MODE_REL, 0);
3024 static void task_clock_perf_counter_disable(struct perf_counter *counter)
3026 if (counter->hw.irq_period)
3027 hrtimer_cancel(&counter->hw.hrtimer);
3028 task_clock_perf_counter_update(counter, counter->ctx->time);
3032 static void task_clock_perf_counter_read(struct perf_counter *counter)
3037 update_context_time(counter->ctx);
3038 time = counter->ctx->time;
3040 u64 now = perf_clock();
3041 u64 delta = now - counter->ctx->timestamp;
3042 time = counter->ctx->time + delta;
3045 task_clock_perf_counter_update(counter, time);
3048 static const struct pmu perf_ops_task_clock = {
3049 .enable = task_clock_perf_counter_enable,
3050 .disable = task_clock_perf_counter_disable,
3051 .read = task_clock_perf_counter_read,
3055 * Software counter: cpu migrations
3058 static inline u64 get_cpu_migrations(struct perf_counter *counter)
3060 struct task_struct *curr = counter->ctx->task;
3063 return curr->se.nr_migrations;
3064 return cpu_nr_migrations(smp_processor_id());
3067 static void cpu_migrations_perf_counter_update(struct perf_counter *counter)
3072 prev = atomic64_read(&counter->hw.prev_count);
3073 now = get_cpu_migrations(counter);
3075 atomic64_set(&counter->hw.prev_count, now);
3079 atomic64_add(delta, &counter->count);
3082 static void cpu_migrations_perf_counter_read(struct perf_counter *counter)
3084 cpu_migrations_perf_counter_update(counter);
3087 static int cpu_migrations_perf_counter_enable(struct perf_counter *counter)
3089 if (counter->prev_state <= PERF_COUNTER_STATE_OFF)
3090 atomic64_set(&counter->hw.prev_count,
3091 get_cpu_migrations(counter));
3095 static void cpu_migrations_perf_counter_disable(struct perf_counter *counter)
3097 cpu_migrations_perf_counter_update(counter);
3100 static const struct pmu perf_ops_cpu_migrations = {
3101 .enable = cpu_migrations_perf_counter_enable,
3102 .disable = cpu_migrations_perf_counter_disable,
3103 .read = cpu_migrations_perf_counter_read,
3106 #ifdef CONFIG_EVENT_PROFILE
3107 void perf_tpcounter_event(int event_id)
3109 struct pt_regs *regs = get_irq_regs();
3112 regs = task_pt_regs(current);
3114 __perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, regs, 0);
3116 EXPORT_SYMBOL_GPL(perf_tpcounter_event);
3118 extern int ftrace_profile_enable(int);
3119 extern void ftrace_profile_disable(int);
3121 static void tp_perf_counter_destroy(struct perf_counter *counter)
3123 ftrace_profile_disable(perf_event_id(&counter->hw_event));
3126 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
3128 int event_id = perf_event_id(&counter->hw_event);
3131 ret = ftrace_profile_enable(event_id);
3135 counter->destroy = tp_perf_counter_destroy;
3136 counter->hw.irq_period = counter->hw_event.irq_period;
3138 return &perf_ops_generic;
3141 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
3147 static const struct pmu *sw_perf_counter_init(struct perf_counter *counter)
3149 const struct pmu *pmu = NULL;
3152 * Software counters (currently) can't in general distinguish
3153 * between user, kernel and hypervisor events.
3154 * However, context switches and cpu migrations are considered
3155 * to be kernel events, and page faults are never hypervisor
3158 switch (perf_event_id(&counter->hw_event)) {
3159 case PERF_COUNT_CPU_CLOCK:
3160 pmu = &perf_ops_cpu_clock;
3163 case PERF_COUNT_TASK_CLOCK:
3165 * If the user instantiates this as a per-cpu counter,
3166 * use the cpu_clock counter instead.
3168 if (counter->ctx->task)
3169 pmu = &perf_ops_task_clock;
3171 pmu = &perf_ops_cpu_clock;
3174 case PERF_COUNT_PAGE_FAULTS:
3175 case PERF_COUNT_PAGE_FAULTS_MIN:
3176 case PERF_COUNT_PAGE_FAULTS_MAJ:
3177 case PERF_COUNT_CONTEXT_SWITCHES:
3178 pmu = &perf_ops_generic;
3180 case PERF_COUNT_CPU_MIGRATIONS:
3181 if (!counter->hw_event.exclude_kernel)
3182 pmu = &perf_ops_cpu_migrations;
3190 * Allocate and initialize a counter structure
3192 static struct perf_counter *
3193 perf_counter_alloc(struct perf_counter_hw_event *hw_event,
3195 struct perf_counter_context *ctx,
3196 struct perf_counter *group_leader,
3199 const struct pmu *pmu;
3200 struct perf_counter *counter;
3201 struct hw_perf_counter *hwc;
3204 counter = kzalloc(sizeof(*counter), gfpflags);
3206 return ERR_PTR(-ENOMEM);
3209 * Single counters are their own group leaders, with an
3210 * empty sibling list:
3213 group_leader = counter;
3215 mutex_init(&counter->child_mutex);
3216 INIT_LIST_HEAD(&counter->child_list);
3218 INIT_LIST_HEAD(&counter->list_entry);
3219 INIT_LIST_HEAD(&counter->event_entry);
3220 INIT_LIST_HEAD(&counter->sibling_list);
3221 init_waitqueue_head(&counter->waitq);
3223 mutex_init(&counter->mmap_mutex);
3226 counter->hw_event = *hw_event;
3227 counter->group_leader = group_leader;
3228 counter->pmu = NULL;
3230 counter->oncpu = -1;
3232 counter->state = PERF_COUNTER_STATE_INACTIVE;
3233 if (hw_event->disabled)
3234 counter->state = PERF_COUNTER_STATE_OFF;
3239 if (hw_event->freq && hw_event->irq_freq)
3240 hwc->irq_period = div64_u64(TICK_NSEC, hw_event->irq_freq);
3242 hwc->irq_period = hw_event->irq_period;
3245 * we currently do not support PERF_RECORD_GROUP on inherited counters
3247 if (hw_event->inherit && (hw_event->record_type & PERF_RECORD_GROUP))
3250 if (perf_event_raw(hw_event)) {
3251 pmu = hw_perf_counter_init(counter);
3255 switch (perf_event_type(hw_event)) {
3256 case PERF_TYPE_HARDWARE:
3257 pmu = hw_perf_counter_init(counter);
3260 case PERF_TYPE_SOFTWARE:
3261 pmu = sw_perf_counter_init(counter);
3264 case PERF_TYPE_TRACEPOINT:
3265 pmu = tp_perf_counter_init(counter);
3272 else if (IS_ERR(pmu))
3277 return ERR_PTR(err);
3282 atomic_inc(&nr_counters);
3283 if (counter->hw_event.mmap)
3284 atomic_inc(&nr_mmap_tracking);
3285 if (counter->hw_event.munmap)
3286 atomic_inc(&nr_munmap_tracking);
3287 if (counter->hw_event.comm)
3288 atomic_inc(&nr_comm_tracking);
3294 * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
3296 * @hw_event_uptr: event type attributes for monitoring/sampling
3299 * @group_fd: group leader counter fd
3301 SYSCALL_DEFINE5(perf_counter_open,
3302 const struct perf_counter_hw_event __user *, hw_event_uptr,
3303 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
3305 struct perf_counter *counter, *group_leader;
3306 struct perf_counter_hw_event hw_event;
3307 struct perf_counter_context *ctx;
3308 struct file *counter_file = NULL;
3309 struct file *group_file = NULL;
3310 int fput_needed = 0;
3311 int fput_needed2 = 0;
3314 /* for future expandability... */
3318 if (copy_from_user(&hw_event, hw_event_uptr, sizeof(hw_event)) != 0)
3322 * Get the target context (task or percpu):
3324 ctx = find_get_context(pid, cpu);
3326 return PTR_ERR(ctx);
3329 * Look up the group leader (we will attach this counter to it):
3331 group_leader = NULL;
3332 if (group_fd != -1) {
3334 group_file = fget_light(group_fd, &fput_needed);
3336 goto err_put_context;
3337 if (group_file->f_op != &perf_fops)
3338 goto err_put_context;
3340 group_leader = group_file->private_data;
3342 * Do not allow a recursive hierarchy (this new sibling
3343 * becoming part of another group-sibling):
3345 if (group_leader->group_leader != group_leader)
3346 goto err_put_context;
3348 * Do not allow to attach to a group in a different
3349 * task or CPU context:
3351 if (group_leader->ctx != ctx)
3352 goto err_put_context;
3354 * Only a group leader can be exclusive or pinned
3356 if (hw_event.exclusive || hw_event.pinned)
3357 goto err_put_context;
3360 counter = perf_counter_alloc(&hw_event, cpu, ctx, group_leader,
3362 ret = PTR_ERR(counter);
3363 if (IS_ERR(counter))
3364 goto err_put_context;
3366 ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
3368 goto err_free_put_context;
3370 counter_file = fget_light(ret, &fput_needed2);
3372 goto err_free_put_context;
3374 counter->filp = counter_file;
3375 WARN_ON_ONCE(ctx->parent_ctx);
3376 mutex_lock(&ctx->mutex);
3377 perf_install_in_context(ctx, counter, cpu);
3379 mutex_unlock(&ctx->mutex);
3381 counter->owner = current;
3382 get_task_struct(current);
3383 mutex_lock(¤t->perf_counter_mutex);
3384 list_add_tail(&counter->owner_entry, ¤t->perf_counter_list);
3385 mutex_unlock(¤t->perf_counter_mutex);
3387 fput_light(counter_file, fput_needed2);
3390 fput_light(group_file, fput_needed);
3394 err_free_put_context:
3404 * inherit a counter from parent task to child task:
3406 static struct perf_counter *
3407 inherit_counter(struct perf_counter *parent_counter,
3408 struct task_struct *parent,
3409 struct perf_counter_context *parent_ctx,
3410 struct task_struct *child,
3411 struct perf_counter *group_leader,
3412 struct perf_counter_context *child_ctx)
3414 struct perf_counter *child_counter;
3417 * Instead of creating recursive hierarchies of counters,
3418 * we link inherited counters back to the original parent,
3419 * which has a filp for sure, which we use as the reference
3422 if (parent_counter->parent)
3423 parent_counter = parent_counter->parent;
3425 child_counter = perf_counter_alloc(&parent_counter->hw_event,
3426 parent_counter->cpu, child_ctx,
3427 group_leader, GFP_KERNEL);
3428 if (IS_ERR(child_counter))
3429 return child_counter;
3433 * Make the child state follow the state of the parent counter,
3434 * not its hw_event.disabled bit. We hold the parent's mutex,
3435 * so we won't race with perf_counter_{en,dis}able_family.
3437 if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
3438 child_counter->state = PERF_COUNTER_STATE_INACTIVE;
3440 child_counter->state = PERF_COUNTER_STATE_OFF;
3443 * Link it up in the child's context:
3445 add_counter_to_ctx(child_counter, child_ctx);
3447 child_counter->parent = parent_counter;
3449 * inherit into child's child as well:
3451 child_counter->hw_event.inherit = 1;
3454 * Get a reference to the parent filp - we will fput it
3455 * when the child counter exits. This is safe to do because
3456 * we are in the parent and we know that the filp still
3457 * exists and has a nonzero count:
3459 atomic_long_inc(&parent_counter->filp->f_count);
3462 * Link this into the parent counter's child list
3464 WARN_ON_ONCE(parent_counter->ctx->parent_ctx);
3465 mutex_lock(&parent_counter->child_mutex);
3466 list_add_tail(&child_counter->child_list, &parent_counter->child_list);
3467 mutex_unlock(&parent_counter->child_mutex);
3469 return child_counter;
3472 static int inherit_group(struct perf_counter *parent_counter,
3473 struct task_struct *parent,
3474 struct perf_counter_context *parent_ctx,
3475 struct task_struct *child,
3476 struct perf_counter_context *child_ctx)
3478 struct perf_counter *leader;
3479 struct perf_counter *sub;
3480 struct perf_counter *child_ctr;
3482 leader = inherit_counter(parent_counter, parent, parent_ctx,
3483 child, NULL, child_ctx);
3485 return PTR_ERR(leader);
3486 list_for_each_entry(sub, &parent_counter->sibling_list, list_entry) {
3487 child_ctr = inherit_counter(sub, parent, parent_ctx,
3488 child, leader, child_ctx);
3489 if (IS_ERR(child_ctr))
3490 return PTR_ERR(child_ctr);
3495 static void sync_child_counter(struct perf_counter *child_counter,
3496 struct perf_counter *parent_counter)
3500 child_val = atomic64_read(&child_counter->count);
3503 * Add back the child's count to the parent's count:
3505 atomic64_add(child_val, &parent_counter->count);
3506 atomic64_add(child_counter->total_time_enabled,
3507 &parent_counter->child_total_time_enabled);
3508 atomic64_add(child_counter->total_time_running,
3509 &parent_counter->child_total_time_running);
3512 * Remove this counter from the parent's list
3514 WARN_ON_ONCE(parent_counter->ctx->parent_ctx);
3515 mutex_lock(&parent_counter->child_mutex);
3516 list_del_init(&child_counter->child_list);
3517 mutex_unlock(&parent_counter->child_mutex);
3520 * Release the parent counter, if this was the last
3523 fput(parent_counter->filp);
3527 __perf_counter_exit_task(struct task_struct *child,
3528 struct perf_counter *child_counter,
3529 struct perf_counter_context *child_ctx)
3531 struct perf_counter *parent_counter;
3533 update_counter_times(child_counter);
3534 perf_counter_remove_from_context(child_counter);
3536 parent_counter = child_counter->parent;
3538 * It can happen that parent exits first, and has counters
3539 * that are still around due to the child reference. These
3540 * counters need to be zapped - but otherwise linger.
3542 if (parent_counter) {
3543 sync_child_counter(child_counter, parent_counter);
3544 free_counter(child_counter);
3549 * When a child task exits, feed back counter values to parent counters.
3551 void perf_counter_exit_task(struct task_struct *child)
3553 struct perf_counter *child_counter, *tmp;
3554 struct perf_counter_context *child_ctx;
3555 unsigned long flags;
3557 if (likely(!child->perf_counter_ctxp))
3560 local_irq_save(flags);
3562 * We can't reschedule here because interrupts are disabled,
3563 * and either child is current or it is a task that can't be
3564 * scheduled, so we are now safe from rescheduling changing
3567 child_ctx = child->perf_counter_ctxp;
3568 __perf_counter_task_sched_out(child_ctx);
3571 * Take the context lock here so that if find_get_context is
3572 * reading child->perf_counter_ctxp, we wait until it has
3573 * incremented the context's refcount before we do put_ctx below.
3575 spin_lock(&child_ctx->lock);
3576 child->perf_counter_ctxp = NULL;
3577 if (child_ctx->parent_ctx) {
3579 * This context is a clone; unclone it so it can't get
3580 * swapped to another process while we're removing all
3581 * the counters from it.
3583 put_ctx(child_ctx->parent_ctx);
3584 child_ctx->parent_ctx = NULL;
3586 spin_unlock(&child_ctx->lock);
3587 local_irq_restore(flags);
3589 mutex_lock(&child_ctx->mutex);
3592 list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
3594 __perf_counter_exit_task(child, child_counter, child_ctx);
3597 * If the last counter was a group counter, it will have appended all
3598 * its siblings to the list, but we obtained 'tmp' before that which
3599 * will still point to the list head terminating the iteration.
3601 if (!list_empty(&child_ctx->counter_list))
3604 mutex_unlock(&child_ctx->mutex);
3610 * Initialize the perf_counter context in task_struct
3612 int perf_counter_init_task(struct task_struct *child)
3614 struct perf_counter_context *child_ctx, *parent_ctx;
3615 struct perf_counter_context *cloned_ctx;
3616 struct perf_counter *counter;
3617 struct task_struct *parent = current;
3618 int inherited_all = 1;
3622 child->perf_counter_ctxp = NULL;
3624 mutex_init(&child->perf_counter_mutex);
3625 INIT_LIST_HEAD(&child->perf_counter_list);
3627 if (likely(!parent->perf_counter_ctxp))
3631 * This is executed from the parent task context, so inherit
3632 * counters that have been marked for cloning.
3633 * First allocate and initialize a context for the child.
3636 child_ctx = kmalloc(sizeof(struct perf_counter_context), GFP_KERNEL);
3640 __perf_counter_init_context(child_ctx, child);
3641 child->perf_counter_ctxp = child_ctx;
3642 get_task_struct(child);
3645 * If the parent's context is a clone, temporarily set its
3646 * parent_gen to an impossible value (all 1s) so it won't get
3647 * swapped under us. The rcu_read_lock makes sure that
3648 * parent_ctx continues to exist even if it gets swapped to
3649 * another process and then freed while we are trying to get
3654 parent_ctx = rcu_dereference(parent->perf_counter_ctxp);
3656 * No need to check if parent_ctx != NULL here; since we saw
3657 * it non-NULL earlier, the only reason for it to become NULL
3658 * is if we exit, and since we're currently in the middle of
3659 * a fork we can't be exiting at the same time.
3661 spin_lock_irq(&parent_ctx->lock);
3662 if (parent_ctx != rcu_dereference(parent->perf_counter_ctxp)) {
3663 spin_unlock_irq(&parent_ctx->lock);
3666 cloned_gen = parent_ctx->parent_gen;
3667 if (parent_ctx->parent_ctx)
3668 parent_ctx->parent_gen = ~0ull;
3669 spin_unlock_irq(&parent_ctx->lock);
3673 * Lock the parent list. No need to lock the child - not PID
3674 * hashed yet and not running, so nobody can access it.
3676 mutex_lock(&parent_ctx->mutex);
3679 * We dont have to disable NMIs - we are only looking at
3680 * the list, not manipulating it:
3682 list_for_each_entry_rcu(counter, &parent_ctx->event_list, event_entry) {
3683 if (counter != counter->group_leader)
3686 if (!counter->hw_event.inherit) {
3691 ret = inherit_group(counter, parent, parent_ctx,
3699 if (inherited_all) {
3701 * Mark the child context as a clone of the parent
3702 * context, or of whatever the parent is a clone of.
3703 * Note that if the parent is a clone, it could get
3704 * uncloned at any point, but that doesn't matter
3705 * because the list of counters and the generation
3706 * count can't have changed since we took the mutex.
3708 cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
3710 child_ctx->parent_ctx = cloned_ctx;
3711 child_ctx->parent_gen = cloned_gen;
3713 child_ctx->parent_ctx = parent_ctx;
3714 child_ctx->parent_gen = parent_ctx->generation;
3716 get_ctx(child_ctx->parent_ctx);
3719 mutex_unlock(&parent_ctx->mutex);
3722 * Restore the clone status of the parent.
3724 if (parent_ctx->parent_ctx) {
3725 spin_lock_irq(&parent_ctx->lock);
3726 if (parent_ctx->parent_ctx)
3727 parent_ctx->parent_gen = cloned_gen;
3728 spin_unlock_irq(&parent_ctx->lock);
3734 static void __cpuinit perf_counter_init_cpu(int cpu)
3736 struct perf_cpu_context *cpuctx;
3738 cpuctx = &per_cpu(perf_cpu_context, cpu);
3739 __perf_counter_init_context(&cpuctx->ctx, NULL);
3741 spin_lock(&perf_resource_lock);
3742 cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
3743 spin_unlock(&perf_resource_lock);
3745 hw_perf_counter_setup(cpu);
3748 #ifdef CONFIG_HOTPLUG_CPU
3749 static void __perf_counter_exit_cpu(void *info)
3751 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
3752 struct perf_counter_context *ctx = &cpuctx->ctx;
3753 struct perf_counter *counter, *tmp;
3755 list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
3756 __perf_counter_remove_from_context(counter);
3758 static void perf_counter_exit_cpu(int cpu)
3760 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
3761 struct perf_counter_context *ctx = &cpuctx->ctx;
3763 mutex_lock(&ctx->mutex);
3764 smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
3765 mutex_unlock(&ctx->mutex);
3768 static inline void perf_counter_exit_cpu(int cpu) { }
3771 static int __cpuinit
3772 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
3774 unsigned int cpu = (long)hcpu;
3778 case CPU_UP_PREPARE:
3779 case CPU_UP_PREPARE_FROZEN:
3780 perf_counter_init_cpu(cpu);
3783 case CPU_DOWN_PREPARE:
3784 case CPU_DOWN_PREPARE_FROZEN:
3785 perf_counter_exit_cpu(cpu);
3795 static struct notifier_block __cpuinitdata perf_cpu_nb = {
3796 .notifier_call = perf_cpu_notify,
3799 void __init perf_counter_init(void)
3801 perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
3802 (void *)(long)smp_processor_id());
3803 register_cpu_notifier(&perf_cpu_nb);
3806 static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
3808 return sprintf(buf, "%d\n", perf_reserved_percpu);
3812 perf_set_reserve_percpu(struct sysdev_class *class,
3816 struct perf_cpu_context *cpuctx;
3820 err = strict_strtoul(buf, 10, &val);
3823 if (val > perf_max_counters)
3826 spin_lock(&perf_resource_lock);
3827 perf_reserved_percpu = val;
3828 for_each_online_cpu(cpu) {
3829 cpuctx = &per_cpu(perf_cpu_context, cpu);
3830 spin_lock_irq(&cpuctx->ctx.lock);
3831 mpt = min(perf_max_counters - cpuctx->ctx.nr_counters,
3832 perf_max_counters - perf_reserved_percpu);
3833 cpuctx->max_pertask = mpt;
3834 spin_unlock_irq(&cpuctx->ctx.lock);
3836 spin_unlock(&perf_resource_lock);
3841 static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
3843 return sprintf(buf, "%d\n", perf_overcommit);
3847 perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
3852 err = strict_strtoul(buf, 10, &val);
3858 spin_lock(&perf_resource_lock);
3859 perf_overcommit = val;
3860 spin_unlock(&perf_resource_lock);
3865 static SYSDEV_CLASS_ATTR(
3868 perf_show_reserve_percpu,
3869 perf_set_reserve_percpu
3872 static SYSDEV_CLASS_ATTR(
3875 perf_show_overcommit,
3879 static struct attribute *perfclass_attrs[] = {
3880 &attr_reserve_percpu.attr,
3881 &attr_overcommit.attr,
3885 static struct attribute_group perfclass_attr_group = {
3886 .attrs = perfclass_attrs,
3887 .name = "perf_counters",
3890 static int __init perf_counter_sysfs_init(void)
3892 return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
3893 &perfclass_attr_group);
3895 device_initcall(perf_counter_sysfs_init);