perf_counter: Rename list_entry -> group_entry, counter_list -> group_list

[profile/ivi/kernel-adaptation-intel-automotive.git] / kernel / perf_counter.c

diff --git a/kernel/perf_counter.c b/kernel/perf_counter.c
index e0d91fd..13ad73a 100644 (file)
--- a/kernel/perf_counter.c
+++ b/kernel/perf_counter.c
@@ -106,16 +106,16 @@ hw_perf_group_sched_in(struct perf_counter *group_leader,
 
 void __weak perf_counter_print_debug(void)     { }
 
 
 void __weak perf_counter_print_debug(void)     { }
 

-static DEFINE_PER_CPU(int, disable_count);
+static DEFINE_PER_CPU(int, perf_disable_count);

 
 void __perf_disable(void)
 {
 
 void __perf_disable(void)
 {

-       __get_cpu_var(disable_count)++;
+       __get_cpu_var(perf_disable_count)++;

 }
 
 bool __perf_enable(void)
 {
 }
 
 bool __perf_enable(void)
 {

-       return !--__get_cpu_var(disable_count);
+       return !--__get_cpu_var(perf_disable_count);

 }
 
 void perf_disable(void)
 }
 
 void perf_disable(void)


@@ -258,9 +258,9 @@ list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
         * leader's sibling list:
         */
        if (group_leader == counter)
         * leader's sibling list:
         */
        if (group_leader == counter)

-               list_add_tail(&counter->list_entry, &ctx->counter_list);
+               list_add_tail(&counter->group_entry, &ctx->group_list);

        else {
        else {

-               list_add_tail(&counter->list_entry, &group_leader->sibling_list);
+               list_add_tail(&counter->group_entry, &group_leader->sibling_list);

                group_leader->nr_siblings++;
        }
 
                group_leader->nr_siblings++;
        }
 


@@ -279,13 +279,13 @@ list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
 {
        struct perf_counter *sibling, *tmp;
 
 {
        struct perf_counter *sibling, *tmp;
 

-       if (list_empty(&counter->list_entry))
+       if (list_empty(&counter->group_entry))

                return;
        ctx->nr_counters--;
        if (counter->attr.inherit_stat)
                ctx->nr_stat--;
 
                return;
        ctx->nr_counters--;
        if (counter->attr.inherit_stat)
                ctx->nr_stat--;
 

-       list_del_init(&counter->list_entry);
+       list_del_init(&counter->group_entry);

        list_del_rcu(&counter->event_entry);
 
        if (counter->group_leader != counter)
        list_del_rcu(&counter->event_entry);
 
        if (counter->group_leader != counter)


@@ -296,10 +296,9 @@ list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
         * upgrade the siblings to singleton counters by adding them
         * to the context list directly:
         */
         * upgrade the siblings to singleton counters by adding them
         * to the context list directly:
         */

-       list_for_each_entry_safe(sibling, tmp,
-                                &counter->sibling_list, list_entry) {
+       list_for_each_entry_safe(sibling, tmp, &counter->sibling_list, group_entry) {

 
 

-               list_move_tail(&sibling->list_entry, &ctx->counter_list);
+               list_move_tail(&sibling->group_entry, &ctx->group_list);

                sibling->group_leader = sibling;
        }
 }
                sibling->group_leader = sibling;
        }
 }


@@ -343,7 +342,7 @@ group_sched_out(struct perf_counter *group_counter,
        /*
         * Schedule out siblings (if any):
         */
        /*
         * Schedule out siblings (if any):
         */

-       list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
+       list_for_each_entry(counter, &group_counter->sibling_list, group_entry)

                counter_sched_out(counter, cpuctx, ctx);
 
        if (group_counter->attr.exclusive)
                counter_sched_out(counter, cpuctx, ctx);
 
        if (group_counter->attr.exclusive)


@@ -435,7 +434,7 @@ retry:
        /*
         * If the context is active we need to retry the smp call.
         */
        /*
         * If the context is active we need to retry the smp call.
         */

-       if (ctx->nr_active && !list_empty(&counter->list_entry)) {
+       if (ctx->nr_active && !list_empty(&counter->group_entry)) {

                spin_unlock_irq(&ctx->lock);
                goto retry;
        }
                spin_unlock_irq(&ctx->lock);
                goto retry;
        }


@@ -445,7 +444,7 @@ retry:
         * can remove the counter safely, if the call above did not
         * succeed.
         */
         * can remove the counter safely, if the call above did not
         * succeed.
         */

-       if (!list_empty(&counter->list_entry)) {
+       if (!list_empty(&counter->group_entry)) {

                list_del_counter(counter, ctx);
        }
        spin_unlock_irq(&ctx->lock);
                list_del_counter(counter, ctx);
        }
        spin_unlock_irq(&ctx->lock);


@@ -497,7 +496,7 @@ static void update_group_times(struct perf_counter *leader)
        struct perf_counter *counter;
 
        update_counter_times(leader);
        struct perf_counter *counter;
 
        update_counter_times(leader);

-       list_for_each_entry(counter, &leader->sibling_list, list_entry)
+       list_for_each_entry(counter, &leader->sibling_list, group_entry)

                update_counter_times(counter);
 }
 
                update_counter_times(counter);
 }
 


@@ -643,7 +642,7 @@ group_sched_in(struct perf_counter *group_counter,
        /*
         * Schedule in siblings as one group (if any):
         */
        /*
         * Schedule in siblings as one group (if any):
         */

-       list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
+       list_for_each_entry(counter, &group_counter->sibling_list, group_entry) {

                if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
                        partial_group = counter;
                        goto group_error;
                if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
                        partial_group = counter;
                        goto group_error;


@@ -657,7 +656,7 @@ group_error:
         * Groups can be scheduled in as one unit only, so undo any
         * partial group before returning:
         */
         * Groups can be scheduled in as one unit only, so undo any
         * partial group before returning:
         */

-       list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
+       list_for_each_entry(counter, &group_counter->sibling_list, group_entry) {

                if (counter == partial_group)
                        break;
                counter_sched_out(counter, cpuctx, ctx);
                if (counter == partial_group)
                        break;
                counter_sched_out(counter, cpuctx, ctx);


@@ -678,7 +677,7 @@ static int is_software_only_group(struct perf_counter *leader)
        if (!is_software_counter(leader))
                return 0;
 
        if (!is_software_counter(leader))
                return 0;
 

-       list_for_each_entry(counter, &leader->sibling_list, list_entry)
+       list_for_each_entry(counter, &leader->sibling_list, group_entry)

                if (!is_software_counter(counter))
                        return 0;
 
                if (!is_software_counter(counter))
                        return 0;
 


@@ -842,7 +841,7 @@ retry:
        /*
         * we need to retry the smp call.
         */
        /*
         * we need to retry the smp call.
         */

-       if (ctx->is_active && list_empty(&counter->list_entry)) {
+       if (ctx->is_active && list_empty(&counter->group_entry)) {

                spin_unlock_irq(&ctx->lock);
                goto retry;
        }
                spin_unlock_irq(&ctx->lock);
                goto retry;
        }


@@ -852,7 +851,7 @@ retry:
         * can add the counter safely, if it the call above did not
         * succeed.
         */
         * can add the counter safely, if it the call above did not
         * succeed.
         */

-       if (list_empty(&counter->list_entry))
+       if (list_empty(&counter->group_entry))

                add_counter_to_ctx(counter, ctx);
        spin_unlock_irq(&ctx->lock);
 }
                add_counter_to_ctx(counter, ctx);
        spin_unlock_irq(&ctx->lock);
 }


@@ -872,7 +871,7 @@ static void __perf_counter_mark_enabled(struct perf_counter *counter,
 
        counter->state = PERF_COUNTER_STATE_INACTIVE;
        counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
 
        counter->state = PERF_COUNTER_STATE_INACTIVE;
        counter->tstamp_enabled = ctx->time - counter->total_time_enabled;

-       list_for_each_entry(sub, &counter->sibling_list, list_entry)
+       list_for_each_entry(sub, &counter->sibling_list, group_entry)

                if (sub->state >= PERF_COUNTER_STATE_INACTIVE)
                        sub->tstamp_enabled =
                                ctx->time - sub->total_time_enabled;
                if (sub->state >= PERF_COUNTER_STATE_INACTIVE)
                        sub->tstamp_enabled =
                                ctx->time - sub->total_time_enabled;


@@ -1032,7 +1031,7 @@ void __perf_counter_sched_out(struct perf_counter_context *ctx,
 
        perf_disable();
        if (ctx->nr_active) {
 
        perf_disable();
        if (ctx->nr_active) {

-               list_for_each_entry(counter, &ctx->counter_list, list_entry) {
+               list_for_each_entry(counter, &ctx->group_list, group_entry) {

                        if (counter != counter->group_leader)
                                counter_sched_out(counter, cpuctx, ctx);
                        else
                        if (counter != counter->group_leader)
                                counter_sched_out(counter, cpuctx, ctx);
                        else


@@ -1252,7 +1251,7 @@ __perf_counter_sched_in(struct perf_counter_context *ctx,
         * First go through the list and put on any pinned groups
         * in order to give them the best chance of going on.
         */
         * First go through the list and put on any pinned groups
         * in order to give them the best chance of going on.
         */

-       list_for_each_entry(counter, &ctx->counter_list, list_entry) {
+       list_for_each_entry(counter, &ctx->group_list, group_entry) {

                if (counter->state <= PERF_COUNTER_STATE_OFF ||
                    !counter->attr.pinned)
                        continue;
                if (counter->state <= PERF_COUNTER_STATE_OFF ||
                    !counter->attr.pinned)
                        continue;


@@ -1276,7 +1275,7 @@ __perf_counter_sched_in(struct perf_counter_context *ctx,
                }
        }
 
                }
        }
 

-       list_for_each_entry(counter, &ctx->counter_list, list_entry) {
+       list_for_each_entry(counter, &ctx->group_list, group_entry) {

                /*
                 * Ignore counters in OFF or ERROR state, and
                 * ignore pinned counters since we did them already.
                /*
                 * Ignore counters in OFF or ERROR state, and
                 * ignore pinned counters since we did them already.


@@ -1369,7 +1368,7 @@ static void perf_ctx_adjust_freq(struct perf_counter_context *ctx)
        u64 interrupts, freq;
 
        spin_lock(&ctx->lock);
        u64 interrupts, freq;
 
        spin_lock(&ctx->lock);

-       list_for_each_entry(counter, &ctx->counter_list, list_entry) {
+       list_for_each_entry(counter, &ctx->group_list, group_entry) {

                if (counter->state != PERF_COUNTER_STATE_ACTIVE)
                        continue;
 
                if (counter->state != PERF_COUNTER_STATE_ACTIVE)
                        continue;
 


@@ -1441,8 +1440,8 @@ static void rotate_ctx(struct perf_counter_context *ctx)
         * Rotate the first entry last (works just fine for group counters too):
         */
        perf_disable();
         * Rotate the first entry last (works just fine for group counters too):
         */
        perf_disable();

-       list_for_each_entry(counter, &ctx->counter_list, list_entry) {
-               list_move_tail(&counter->list_entry, &ctx->counter_list);
+       list_for_each_entry(counter, &ctx->group_list, group_entry) {
+               list_move_tail(&counter->group_entry, &ctx->group_list);

                break;
        }
        perf_enable();
                break;
        }
        perf_enable();


@@ -1498,7 +1497,7 @@ static void perf_counter_enable_on_exec(struct task_struct *task)
 
        spin_lock(&ctx->lock);
 
 
        spin_lock(&ctx->lock);
 

-       list_for_each_entry(counter, &ctx->counter_list, list_entry) {
+       list_for_each_entry(counter, &ctx->group_list, group_entry) {

                if (!counter->attr.enable_on_exec)
                        continue;
                counter->attr.enable_on_exec = 0;
                if (!counter->attr.enable_on_exec)
                        continue;
                counter->attr.enable_on_exec = 0;


@@ -1575,7 +1574,7 @@ __perf_counter_init_context(struct perf_counter_context *ctx,
        memset(ctx, 0, sizeof(*ctx));
        spin_lock_init(&ctx->lock);
        mutex_init(&ctx->mutex);
        memset(ctx, 0, sizeof(*ctx));
        spin_lock_init(&ctx->lock);
        mutex_init(&ctx->mutex);

-       INIT_LIST_HEAD(&ctx->counter_list);
+       INIT_LIST_HEAD(&ctx->group_list);

        INIT_LIST_HEAD(&ctx->event_list);
        atomic_set(&ctx->refcount, 1);
        ctx->task = task;
        INIT_LIST_HEAD(&ctx->event_list);
        atomic_set(&ctx->refcount, 1);
        ctx->task = task;


@@ -1818,7 +1817,7 @@ static int perf_counter_read_group(struct perf_counter *counter,
 
        size += err;
 
 
        size += err;
 

-       list_for_each_entry(sub, &leader->sibling_list, list_entry) {
+       list_for_each_entry(sub, &leader->sibling_list, group_entry) {

                err = perf_counter_read_entry(sub, read_format,
                                buf + size);
                if (err < 0)
                err = perf_counter_read_entry(sub, read_format,
                                buf + size);
                if (err < 0)


@@ -1948,7 +1947,7 @@ static void perf_counter_for_each(struct perf_counter *counter,
 
        perf_counter_for_each_child(counter, func);
        func(counter);
 
        perf_counter_for_each_child(counter, func);
        func(counter);

-       list_for_each_entry(sibling, &counter->sibling_list, list_entry)
+       list_for_each_entry(sibling, &counter->sibling_list, group_entry)

                perf_counter_for_each_child(counter, func);
        mutex_unlock(&ctx->mutex);
 }
                perf_counter_for_each_child(counter, func);
        mutex_unlock(&ctx->mutex);
 }


@@ -2176,6 +2175,13 @@ static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
        data->nr_pages = nr_pages;
        atomic_set(&data->lock, -1);
 
        data->nr_pages = nr_pages;
        atomic_set(&data->lock, -1);
 

+       if (counter->attr.watermark) {
+               data->watermark = min_t(long, PAGE_SIZE * nr_pages,
+                                     counter->attr.wakeup_watermark);
+       }
+       if (!data->watermark)
+               data->watermark = max(PAGE_SIZE, PAGE_SIZE * nr_pages / 4);
+

        rcu_assign_pointer(counter->data, data);
 
        return 0;
        rcu_assign_pointer(counter->data, data);
 
        return 0;


@@ -2315,7 +2321,8 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma)
        lock_limit >>= PAGE_SHIFT;
        locked = vma->vm_mm->locked_vm + extra;
 
        lock_limit >>= PAGE_SHIFT;
        locked = vma->vm_mm->locked_vm + extra;
 

-       if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
+       if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
+               !capable(CAP_IPC_LOCK)) {

                ret = -EPERM;
                goto unlock;
        }
                ret = -EPERM;
                goto unlock;
        }


@@ -2504,35 +2511,15 @@ __weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
 /*
  * Output
  */
 /*
  * Output
  */

-
-struct perf_output_handle {
-       struct perf_counter     *counter;
-       struct perf_mmap_data   *data;
-       unsigned long           head;
-       unsigned long           offset;
-       int                     nmi;
-       int                     sample;
-       int                     locked;
-       unsigned long           flags;
-};
-
-static bool perf_output_space(struct perf_mmap_data *data,
-                             unsigned int offset, unsigned int head)
+static bool perf_output_space(struct perf_mmap_data *data, unsigned long tail,
+                             unsigned long offset, unsigned long head)

 {
 {

-       unsigned long tail;

        unsigned long mask;
 
        if (!data->writable)
                return true;
 
        mask = (data->nr_pages << PAGE_SHIFT) - 1;
        unsigned long mask;
 
        if (!data->writable)
                return true;
 
        mask = (data->nr_pages << PAGE_SHIFT) - 1;

-       /*
-        * Userspace could choose to issue a mb() before updating the tail
-        * pointer. So that all reads will be completed before the write is
-        * issued.
-        */
-       tail = ACCESS_ONCE(data->user_page->data_tail);
-       smp_rmb();

 
        offset = (offset - tail) & mask;
        head   = (head   - tail) & mask;
 
        offset = (offset - tail) & mask;
        head   = (head   - tail) & mask;


@@ -2633,8 +2620,8 @@ out:
        local_irq_restore(handle->flags);
 }
 
        local_irq_restore(handle->flags);
 }
 

-static void perf_output_copy(struct perf_output_handle *handle,
-                            const void *buf, unsigned int len)
+void perf_output_copy(struct perf_output_handle *handle,
+                     const void *buf, unsigned int len)

 {
        unsigned int pages_mask;
        unsigned int offset;
 {
        unsigned int pages_mask;
        unsigned int offset;


@@ -2669,16 +2656,13 @@ static void perf_output_copy(struct perf_output_handle *handle,
        WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
 }
 
        WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
 }
 

-#define perf_output_put(handle, x) \
-       perf_output_copy((handle), &(x), sizeof(x))
-
-static int perf_output_begin(struct perf_output_handle *handle,
-                            struct perf_counter *counter, unsigned int size,
-                            int nmi, int sample)
+int perf_output_begin(struct perf_output_handle *handle,
+                     struct perf_counter *counter, unsigned int size,
+                     int nmi, int sample)

 {
        struct perf_counter *output_counter;
        struct perf_mmap_data *data;
 {
        struct perf_counter *output_counter;
        struct perf_mmap_data *data;

-       unsigned int offset, head;
+       unsigned long tail, offset, head;

        int have_lost;
        struct {
                struct perf_event_header header;
        int have_lost;
        struct {
                struct perf_event_header header;


@@ -2716,16 +2700,23 @@ static int perf_output_begin(struct perf_output_handle *handle,
        perf_output_lock(handle);
 
        do {
        perf_output_lock(handle);
 
        do {

+               /*
+                * Userspace could choose to issue a mb() before updating the
+                * tail pointer. So that all reads will be completed before the
+                * write is issued.
+                */
+               tail = ACCESS_ONCE(data->user_page->data_tail);
+               smp_rmb();

                offset = head = atomic_long_read(&data->head);
                head += size;
                offset = head = atomic_long_read(&data->head);
                head += size;

-               if (unlikely(!perf_output_space(data, offset, head)))
+               if (unlikely(!perf_output_space(data, tail, offset, head)))

                        goto fail;
        } while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
 
        handle->offset  = offset;
        handle->head    = head;
 
                        goto fail;
        } while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
 
        handle->offset  = offset;
        handle->head    = head;
 

-       if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
+       if (head - tail > data->watermark)

                atomic_set(&data->wakeup, 1);
 
        if (have_lost) {
                atomic_set(&data->wakeup, 1);
 
        if (have_lost) {


@@ -2749,7 +2740,7 @@ out:
        return -ENOSPC;
 }
 
        return -ENOSPC;
 }
 

-static void perf_output_end(struct perf_output_handle *handle)
+void perf_output_end(struct perf_output_handle *handle)

 {
        struct perf_counter *counter = handle->counter;
        struct perf_mmap_data *data = handle->data;
 {
        struct perf_counter *counter = handle->counter;
        struct perf_mmap_data *data = handle->data;


@@ -2840,7 +2831,7 @@ static void perf_output_read_group(struct perf_output_handle *handle,
 
        perf_output_copy(handle, values, n * sizeof(u64));
 
 
        perf_output_copy(handle, values, n * sizeof(u64));
 

-       list_for_each_entry(sub, &leader->sibling_list, list_entry) {
+       list_for_each_entry(sub, &leader->sibling_list, group_entry) {

                n = 0;
 
                if (sub != counter)
                n = 0;
 
                if (sub != counter)


@@ -2863,156 +2854,176 @@ static void perf_output_read(struct perf_output_handle *handle,
                perf_output_read_one(handle, counter);
 }
 
                perf_output_read_one(handle, counter);
 }
 

-void perf_counter_output(struct perf_counter *counter, int nmi,
-                               struct perf_sample_data *data)
+void perf_output_sample(struct perf_output_handle *handle,
+                       struct perf_event_header *header,
+                       struct perf_sample_data *data,
+                       struct perf_counter *counter)

 {
 {

-       int ret;
-       u64 sample_type = counter->attr.sample_type;
-       struct perf_output_handle handle;
-       struct perf_event_header header;
-       u64 ip;
-       struct {
-               u32 pid, tid;
-       } tid_entry;
-       struct perf_callchain_entry *callchain = NULL;
-       int callchain_size = 0;
-       u64 time;
-       struct {
-               u32 cpu, reserved;
-       } cpu_entry;
+       u64 sample_type = data->type;

 
 

-       header.type = PERF_EVENT_SAMPLE;
-       header.size = sizeof(header);
+       perf_output_put(handle, *header);

 
 

-       header.misc = 0;
-       header.misc |= perf_misc_flags(data->regs);
-
-       if (sample_type & PERF_SAMPLE_IP) {
-               ip = perf_instruction_pointer(data->regs);
-               header.size += sizeof(ip);
-       }
-
-       if (sample_type & PERF_SAMPLE_TID) {
-               /* namespace issues */
-               tid_entry.pid = perf_counter_pid(counter, current);
-               tid_entry.tid = perf_counter_tid(counter, current);
-
-               header.size += sizeof(tid_entry);
-       }
+       if (sample_type & PERF_SAMPLE_IP)
+               perf_output_put(handle, data->ip);

 
 

-       if (sample_type & PERF_SAMPLE_TIME) {
-               /*
-                * Maybe do better on x86 and provide cpu_clock_nmi()
-                */
-               time = sched_clock();
+       if (sample_type & PERF_SAMPLE_TID)
+               perf_output_put(handle, data->tid_entry);

 
 

-               header.size += sizeof(u64);
-       }
+       if (sample_type & PERF_SAMPLE_TIME)
+               perf_output_put(handle, data->time);

 
        if (sample_type & PERF_SAMPLE_ADDR)
 
        if (sample_type & PERF_SAMPLE_ADDR)

-               header.size += sizeof(u64);
+               perf_output_put(handle, data->addr);

 
        if (sample_type & PERF_SAMPLE_ID)
 
        if (sample_type & PERF_SAMPLE_ID)

-               header.size += sizeof(u64);
+               perf_output_put(handle, data->id);

 
        if (sample_type & PERF_SAMPLE_STREAM_ID)
 
        if (sample_type & PERF_SAMPLE_STREAM_ID)

-               header.size += sizeof(u64);
-
-       if (sample_type & PERF_SAMPLE_CPU) {
-               header.size += sizeof(cpu_entry);
+               perf_output_put(handle, data->stream_id);

 
 

-               cpu_entry.cpu = raw_smp_processor_id();
-               cpu_entry.reserved = 0;
-       }
+       if (sample_type & PERF_SAMPLE_CPU)
+               perf_output_put(handle, data->cpu_entry);

 
        if (sample_type & PERF_SAMPLE_PERIOD)
 
        if (sample_type & PERF_SAMPLE_PERIOD)

-               header.size += sizeof(u64);
+               perf_output_put(handle, data->period);

 
        if (sample_type & PERF_SAMPLE_READ)
 
        if (sample_type & PERF_SAMPLE_READ)

-               header.size += perf_counter_read_size(counter);
+               perf_output_read(handle, counter);

 
        if (sample_type & PERF_SAMPLE_CALLCHAIN) {
 
        if (sample_type & PERF_SAMPLE_CALLCHAIN) {

-               callchain = perf_callchain(data->regs);
+               if (data->callchain) {
+                       int size = 1;

 
 

-               if (callchain) {
-                       callchain_size = (1 + callchain->nr) * sizeof(u64);
-                       header.size += callchain_size;
-               } else
-                       header.size += sizeof(u64);
+                       if (data->callchain)
+                               size += data->callchain->nr;
+
+                       size *= sizeof(u64);
+
+                       perf_output_copy(handle, data->callchain, size);
+               } else {
+                       u64 nr = 0;
+                       perf_output_put(handle, nr);
+               }

        }
 
        if (sample_type & PERF_SAMPLE_RAW) {
        }
 
        if (sample_type & PERF_SAMPLE_RAW) {

-               int size = sizeof(u32);
+               if (data->raw) {
+                       perf_output_put(handle, data->raw->size);
+                       perf_output_copy(handle, data->raw->data,
+                                        data->raw->size);
+               } else {
+                       struct {
+                               u32     size;
+                               u32     data;
+                       } raw = {
+                               .size = sizeof(u32),
+                               .data = 0,
+                       };
+                       perf_output_put(handle, raw);
+               }
+       }
+}

 
 

-               if (data->raw)
-                       size += data->raw->size;
-               else
-                       size += sizeof(u32);
+void perf_prepare_sample(struct perf_event_header *header,
+                        struct perf_sample_data *data,
+                        struct perf_counter *counter,
+                        struct pt_regs *regs)
+{
+       u64 sample_type = counter->attr.sample_type;

 
 

-               WARN_ON_ONCE(size & (sizeof(u64)-1));
-               header.size += size;
-       }
+       data->type = sample_type;

 
 

-       ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
-       if (ret)
-               return;
+       header->type = PERF_EVENT_SAMPLE;
+       header->size = sizeof(*header);

 
 

-       perf_output_put(&handle, header);
+       header->misc = 0;
+       header->misc |= perf_misc_flags(regs);

 
 

-       if (sample_type & PERF_SAMPLE_IP)
-               perf_output_put(&handle, ip);
+       if (sample_type & PERF_SAMPLE_IP) {
+               data->ip = perf_instruction_pointer(regs);

 
 

-       if (sample_type & PERF_SAMPLE_TID)
-               perf_output_put(&handle, tid_entry);
+               header->size += sizeof(data->ip);
+       }

 
 

-       if (sample_type & PERF_SAMPLE_TIME)
-               perf_output_put(&handle, time);
+       if (sample_type & PERF_SAMPLE_TID) {
+               /* namespace issues */
+               data->tid_entry.pid = perf_counter_pid(counter, current);
+               data->tid_entry.tid = perf_counter_tid(counter, current);
+
+               header->size += sizeof(data->tid_entry);
+       }
+
+       if (sample_type & PERF_SAMPLE_TIME) {
+               data->time = perf_clock();
+
+               header->size += sizeof(data->time);
+       }

 
        if (sample_type & PERF_SAMPLE_ADDR)
 
        if (sample_type & PERF_SAMPLE_ADDR)

-               perf_output_put(&handle, data->addr);
+               header->size += sizeof(data->addr);

 
        if (sample_type & PERF_SAMPLE_ID) {
 
        if (sample_type & PERF_SAMPLE_ID) {

-               u64 id = primary_counter_id(counter);
+               data->id = primary_counter_id(counter);

 
 

-               perf_output_put(&handle, id);
+               header->size += sizeof(data->id);

        }
 
        }
 

-       if (sample_type & PERF_SAMPLE_STREAM_ID)
-               perf_output_put(&handle, counter->id);
+       if (sample_type & PERF_SAMPLE_STREAM_ID) {
+               data->stream_id = counter->id;

 
 

-       if (sample_type & PERF_SAMPLE_CPU)
-               perf_output_put(&handle, cpu_entry);
+               header->size += sizeof(data->stream_id);
+       }
+
+       if (sample_type & PERF_SAMPLE_CPU) {
+               data->cpu_entry.cpu             = raw_smp_processor_id();
+               data->cpu_entry.reserved        = 0;
+
+               header->size += sizeof(data->cpu_entry);
+       }

 
        if (sample_type & PERF_SAMPLE_PERIOD)
 
        if (sample_type & PERF_SAMPLE_PERIOD)

-               perf_output_put(&handle, data->period);
+               header->size += sizeof(data->period);

 
        if (sample_type & PERF_SAMPLE_READ)
 
        if (sample_type & PERF_SAMPLE_READ)

-               perf_output_read(&handle, counter);
+               header->size += perf_counter_read_size(counter);

 
        if (sample_type & PERF_SAMPLE_CALLCHAIN) {
 
        if (sample_type & PERF_SAMPLE_CALLCHAIN) {

-               if (callchain)
-                       perf_output_copy(&handle, callchain, callchain_size);
-               else {
-                       u64 nr = 0;
-                       perf_output_put(&handle, nr);
-               }
+               int size = 1;
+
+               data->callchain = perf_callchain(regs);
+
+               if (data->callchain)
+                       size += data->callchain->nr;
+
+               header->size += size * sizeof(u64);

        }
 
        if (sample_type & PERF_SAMPLE_RAW) {
        }
 
        if (sample_type & PERF_SAMPLE_RAW) {

-               if (data->raw) {
-                       perf_output_put(&handle, data->raw->size);
-                       perf_output_copy(&handle, data->raw->data, data->raw->size);
-               } else {
-                       struct {
-                               u32     size;
-                               u32     data;
-                       } raw = {
-                               .size = sizeof(u32),
-                               .data = 0,
-                       };
-                       perf_output_put(&handle, raw);
-               }
+               int size = sizeof(u32);
+
+               if (data->raw)
+                       size += data->raw->size;
+               else
+                       size += sizeof(u32);
+
+               WARN_ON_ONCE(size & (sizeof(u64)-1));
+               header->size += size;

        }
        }

+}
+
+static void perf_counter_output(struct perf_counter *counter, int nmi,
+                               struct perf_sample_data *data,
+                               struct pt_regs *regs)
+{
+       struct perf_output_handle handle;
+       struct perf_event_header header;
+
+       perf_prepare_sample(&header, data, counter, regs);
+
+       if (perf_output_begin(&handle, counter, header.size, nmi, 1))
+               return;
+
+       perf_output_sample(&handle, &header, data, counter);

 
        perf_output_end(&handle);
 }
 
        perf_output_end(&handle);
 }


@@ -3071,6 +3082,7 @@ struct perf_task_event {
                u32                             ppid;
                u32                             tid;
                u32                             ptid;
                u32                             ppid;
                u32                             tid;
                u32                             ptid;

+               u64                             time;

        } event;
 };
 
        } event;
 };
 


@@ -3078,9 +3090,12 @@ static void perf_counter_task_output(struct perf_counter *counter,
                                     struct perf_task_event *task_event)
 {
        struct perf_output_handle handle;
                                     struct perf_task_event *task_event)
 {
        struct perf_output_handle handle;

-       int size = task_event->event.header.size;
+       int size;

        struct task_struct *task = task_event->task;
        struct task_struct *task = task_event->task;

-       int ret = perf_output_begin(&handle, counter, size, 0, 0);
+       int ret;
+
+       size  = task_event->event.header.size;
+       ret = perf_output_begin(&handle, counter, size, 0, 0);

 
        if (ret)
                return;
 
        if (ret)
                return;


@@ -3091,7 +3106,10 @@ static void perf_counter_task_output(struct perf_counter *counter,
        task_event->event.tid = perf_counter_tid(counter, task);
        task_event->event.ptid = perf_counter_tid(counter, current);
 
        task_event->event.tid = perf_counter_tid(counter, task);
        task_event->event.ptid = perf_counter_tid(counter, current);
 

+       task_event->event.time = perf_clock();
+

        perf_output_put(&handle, task_event->event);
        perf_output_put(&handle, task_event->event);

+

        perf_output_end(&handle);
 }
 
        perf_output_end(&handle);
 }
 


@@ -3473,7 +3491,7 @@ static void perf_log_throttle(struct perf_counter *counter, int enable)
                        .misc = 0,
                        .size = sizeof(throttle_event),
                },
                        .misc = 0,
                        .size = sizeof(throttle_event),
                },

-               .time           = sched_clock(),
+               .time           = perf_clock(),

                .id             = primary_counter_id(counter),
                .stream_id      = counter->id,
        };
                .id             = primary_counter_id(counter),
                .stream_id      = counter->id,
        };


@@ -3493,14 +3511,16 @@ static void perf_log_throttle(struct perf_counter *counter, int enable)
  * Generic counter overflow handling, sampling.
  */
 
  * Generic counter overflow handling, sampling.
  */
 

-int perf_counter_overflow(struct perf_counter *counter, int nmi,
-                         struct perf_sample_data *data)
+static int __perf_counter_overflow(struct perf_counter *counter, int nmi,
+                                  int throttle, struct perf_sample_data *data,
+                                  struct pt_regs *regs)

 {
        int events = atomic_read(&counter->event_limit);
 {
        int events = atomic_read(&counter->event_limit);

-       int throttle = counter->pmu->unthrottle != NULL;

        struct hw_perf_counter *hwc = &counter->hw;
        int ret = 0;
 
        struct hw_perf_counter *hwc = &counter->hw;
        int ret = 0;
 

+       throttle = (throttle && counter->pmu->unthrottle != NULL);
+

        if (!throttle) {
                hwc->interrupts++;
        } else {
        if (!throttle) {
                hwc->interrupts++;
        } else {


@@ -3523,7 +3543,7 @@ int perf_counter_overflow(struct perf_counter *counter, int nmi,
        }
 
        if (counter->attr.freq) {
        }
 
        if (counter->attr.freq) {

-               u64 now = sched_clock();
+               u64 now = perf_clock();

                s64 delta = now - hwc->freq_stamp;
 
                hwc->freq_stamp = now;
                s64 delta = now - hwc->freq_stamp;
 
                hwc->freq_stamp = now;


@@ -3549,10 +3569,17 @@ int perf_counter_overflow(struct perf_counter *counter, int nmi,
                        perf_counter_disable(counter);
        }
 
                        perf_counter_disable(counter);
        }
 

-       perf_counter_output(counter, nmi, data);
+       perf_counter_output(counter, nmi, data, regs);

        return ret;
 }
 
        return ret;
 }
 

+int perf_counter_overflow(struct perf_counter *counter, int nmi,
+                         struct perf_sample_data *data,
+                         struct pt_regs *regs)
+{
+       return __perf_counter_overflow(counter, nmi, 1, data, regs);
+}
+

 /*
  * Generic software counter infrastructure
  */
 /*
  * Generic software counter infrastructure
  */


@@ -3588,9 +3615,11 @@ again:
 }
 
 static void perf_swcounter_overflow(struct perf_counter *counter,
 }
 
 static void perf_swcounter_overflow(struct perf_counter *counter,

-                                   int nmi, struct perf_sample_data *data)
+                                   int nmi, struct perf_sample_data *data,
+                                   struct pt_regs *regs)

 {
        struct hw_perf_counter *hwc = &counter->hw;
 {
        struct hw_perf_counter *hwc = &counter->hw;

+       int throttle = 0;

        u64 overflow;
 
        data->period = counter->hw.last_period;
        u64 overflow;
 
        data->period = counter->hw.last_period;


@@ -3600,13 +3629,15 @@ static void perf_swcounter_overflow(struct perf_counter *counter,
                return;
 
        for (; overflow; overflow--) {
                return;
 
        for (; overflow; overflow--) {

-               if (perf_counter_overflow(counter, nmi, data)) {
+               if (__perf_counter_overflow(counter, nmi, throttle,
+                                           data, regs)) {

                        /*
                         * We inhibit the overflow from happening when
                         * hwc->interrupts == MAX_INTERRUPTS.
                         */
                        break;
                }
                        /*
                         * We inhibit the overflow from happening when
                         * hwc->interrupts == MAX_INTERRUPTS.
                         */
                        break;
                }

+               throttle = 1;

        }
 }
 
        }
 }
 


@@ -3618,7 +3649,8 @@ static void perf_swcounter_unthrottle(struct perf_counter *counter)
 }
 
 static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
 }
 
 static void perf_swcounter_add(struct perf_counter *counter, u64 nr,

-                              int nmi, struct perf_sample_data *data)
+                              int nmi, struct perf_sample_data *data,
+                              struct pt_regs *regs)

 {
        struct hw_perf_counter *hwc = &counter->hw;
 
 {
        struct hw_perf_counter *hwc = &counter->hw;
 


@@ -3627,11 +3659,11 @@ static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
        if (!hwc->sample_period)
                return;
 
        if (!hwc->sample_period)
                return;
 

-       if (!data->regs)
+       if (!regs)

                return;
 
        if (!atomic64_add_negative(nr, &hwc->period_left))
                return;
 
        if (!atomic64_add_negative(nr, &hwc->period_left))

-               perf_swcounter_overflow(counter, nmi, data);
+               perf_swcounter_overflow(counter, nmi, data, regs);

 }
 
 static int perf_swcounter_is_counting(struct perf_counter *counter)
 }
 
 static int perf_swcounter_is_counting(struct perf_counter *counter)


@@ -3690,7 +3722,8 @@ static int perf_swcounter_match(struct perf_counter *counter,
 static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
                                     enum perf_type_id type,
                                     u32 event, u64 nr, int nmi,
 static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
                                     enum perf_type_id type,
                                     u32 event, u64 nr, int nmi,

-                                    struct perf_sample_data *data)
+                                    struct perf_sample_data *data,
+                                    struct pt_regs *regs)

 {
        struct perf_counter *counter;
 
 {
        struct perf_counter *counter;
 


@@ -3699,8 +3732,8 @@ static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
 
        rcu_read_lock();
        list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
 
        rcu_read_lock();
        list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {

-               if (perf_swcounter_match(counter, type, event, data->regs))
-                       perf_swcounter_add(counter, nr, nmi, data);
+               if (perf_swcounter_match(counter, type, event, regs))
+                       perf_swcounter_add(counter, nr, nmi, data, regs);

        }
        rcu_read_unlock();
 }
        }
        rcu_read_unlock();
 }


@@ -3721,7 +3754,8 @@ static int *perf_swcounter_recursion_context(struct perf_cpu_context *cpuctx)
 
 static void do_perf_swcounter_event(enum perf_type_id type, u32 event,
                                    u64 nr, int nmi,
 
 static void do_perf_swcounter_event(enum perf_type_id type, u32 event,
                                    u64 nr, int nmi,

-                                   struct perf_sample_data *data)
+                                   struct perf_sample_data *data,
+                                   struct pt_regs *regs)

 {
        struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
        int *recursion = perf_swcounter_recursion_context(cpuctx);
 {
        struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
        int *recursion = perf_swcounter_recursion_context(cpuctx);


@@ -3734,7 +3768,7 @@ static void do_perf_swcounter_event(enum perf_type_id type, u32 event,
        barrier();
 
        perf_swcounter_ctx_event(&cpuctx->ctx, type, event,
        barrier();
 
        perf_swcounter_ctx_event(&cpuctx->ctx, type, event,

-                                nr, nmi, data);
+                                nr, nmi, data, regs);

        rcu_read_lock();
        /*
         * doesn't really matter which of the child contexts the
        rcu_read_lock();
        /*
         * doesn't really matter which of the child contexts the


@@ -3742,7 +3776,7 @@ static void do_perf_swcounter_event(enum perf_type_id type, u32 event,
         */
        ctx = rcu_dereference(current->perf_counter_ctxp);
        if (ctx)
         */
        ctx = rcu_dereference(current->perf_counter_ctxp);
        if (ctx)

-               perf_swcounter_ctx_event(ctx, type, event, nr, nmi, data);
+               perf_swcounter_ctx_event(ctx, type, event, nr, nmi, data, regs);

        rcu_read_unlock();
 
        barrier();
        rcu_read_unlock();
 
        barrier();


@@ -3756,11 +3790,11 @@ void __perf_swcounter_event(u32 event, u64 nr, int nmi,
                            struct pt_regs *regs, u64 addr)
 {
        struct perf_sample_data data = {
                            struct pt_regs *regs, u64 addr)
 {
        struct perf_sample_data data = {

-               .regs = regs,

                .addr = addr,
        };
 
                .addr = addr,
        };
 

-       do_perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, &data);
+       do_perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi,
+                               &data, regs);

 }
 
 static void perf_swcounter_read(struct perf_counter *counter)
 }
 
 static void perf_swcounter_read(struct perf_counter *counter)


@@ -3797,6 +3831,7 @@ static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
 {
        enum hrtimer_restart ret = HRTIMER_RESTART;
        struct perf_sample_data data;
 {
        enum hrtimer_restart ret = HRTIMER_RESTART;
        struct perf_sample_data data;

+       struct pt_regs *regs;

        struct perf_counter *counter;
        u64 period;
 
        struct perf_counter *counter;
        u64 period;
 


@@ -3804,17 +3839,17 @@ static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
        counter->pmu->read(counter);
 
        data.addr = 0;
        counter->pmu->read(counter);
 
        data.addr = 0;

-       data.regs = get_irq_regs();
+       regs = get_irq_regs();

        /*
         * In case we exclude kernel IPs or are somehow not in interrupt
         * context, provide the next best thing, the user IP.
         */
        /*
         * In case we exclude kernel IPs or are somehow not in interrupt
         * context, provide the next best thing, the user IP.
         */

-       if ((counter->attr.exclude_kernel || !data.regs) &&
+       if ((counter->attr.exclude_kernel || !regs) &&

                        !counter->attr.exclude_user)
                        !counter->attr.exclude_user)

-               data.regs = task_pt_regs(current);
+               regs = task_pt_regs(current);

 
 

-       if (data.regs) {
-               if (perf_counter_overflow(counter, 0, &data))
+       if (regs) {
+               if (perf_counter_overflow(counter, 0, &data, regs))

                        ret = HRTIMER_NORESTART;
        }
 
                        ret = HRTIMER_NORESTART;
        }
 


@@ -3950,15 +3985,17 @@ void perf_tpcounter_event(int event_id, u64 addr, u64 count, void *record,
        };
 
        struct perf_sample_data data = {
        };
 
        struct perf_sample_data data = {

-               .regs = get_irq_regs(),

                .addr = addr,
                .raw = &raw,
        };
 
                .addr = addr,
                .raw = &raw,
        };
 

-       if (!data.regs)
-               data.regs = task_pt_regs(current);
+       struct pt_regs *regs = get_irq_regs();
+
+       if (!regs)
+               regs = task_pt_regs(current);

 
 

-       do_perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, count, 1, &data);
+       do_perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
+                               &data, regs);

 }
 EXPORT_SYMBOL_GPL(perf_tpcounter_event);
 
 }
 EXPORT_SYMBOL_GPL(perf_tpcounter_event);
 


@@ -4080,7 +4117,7 @@ perf_counter_alloc(struct perf_counter_attr *attr,
        mutex_init(&counter->child_mutex);
        INIT_LIST_HEAD(&counter->child_list);
 
        mutex_init(&counter->child_mutex);
        INIT_LIST_HEAD(&counter->child_list);
 

-       INIT_LIST_HEAD(&counter->list_entry);
+       INIT_LIST_HEAD(&counter->group_entry);

        INIT_LIST_HEAD(&counter->event_entry);
        INIT_LIST_HEAD(&counter->sibling_list);
        init_waitqueue_head(&counter->waitq);
        INIT_LIST_HEAD(&counter->event_entry);
        INIT_LIST_HEAD(&counter->sibling_list);
        init_waitqueue_head(&counter->waitq);


@@ -4170,8 +4207,8 @@ done:
 static int perf_copy_attr(struct perf_counter_attr __user *uattr,
                          struct perf_counter_attr *attr)
 {
 static int perf_copy_attr(struct perf_counter_attr __user *uattr,
                          struct perf_counter_attr *attr)
 {

-       int ret;

        u32 size;
        u32 size;

+       int ret;

 
        if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
                return -EFAULT;
 
        if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
                return -EFAULT;


@@ -4196,25 +4233,26 @@ static int perf_copy_attr(struct perf_counter_attr __user *uattr,
 
        /*
         * If we're handed a bigger struct than we know of,
 
        /*
         * If we're handed a bigger struct than we know of,

-        * ensure all the unknown bits are 0.
+        * ensure all the unknown bits are 0 - i.e. new
+        * user-space does not rely on any kernel feature
+        * extensions we dont know about yet.

         */
        if (size > sizeof(*attr)) {
         */
        if (size > sizeof(*attr)) {

-               unsigned long val;
-               unsigned long __user *addr;
-               unsigned long __user *end;
+               unsigned char __user *addr;
+               unsigned char __user *end;
+               unsigned char val;

 
 

-               addr = PTR_ALIGN((void __user *)uattr + sizeof(*attr),
-                               sizeof(unsigned long));
-               end  = PTR_ALIGN((void __user *)uattr + size,
-                               sizeof(unsigned long));
+               addr = (void __user *)uattr + sizeof(*attr);
+               end  = (void __user *)uattr + size;

 
 

-               for (; addr < end; addr += sizeof(unsigned long)) {
+               for (; addr < end; addr++) {

                        ret = get_user(val, addr);
                        if (ret)
                                return ret;
                        if (val)
                                goto err_size;
                }
                        ret = get_user(val, addr);
                        if (ret)
                                return ret;
                        if (val)
                                goto err_size;
                }

+               size = sizeof(*attr);

        }
 
        ret = copy_from_user(attr, uattr, size);
        }
 
        ret = copy_from_user(attr, uattr, size);


@@ -4505,7 +4543,7 @@ static int inherit_group(struct perf_counter *parent_counter,
                                 child, NULL, child_ctx);
        if (IS_ERR(leader))
                return PTR_ERR(leader);
                                 child, NULL, child_ctx);
        if (IS_ERR(leader))
                return PTR_ERR(leader);

-       list_for_each_entry(sub, &parent_counter->sibling_list, list_entry) {
+       list_for_each_entry(sub, &parent_counter->sibling_list, group_entry) {

                child_ctr = inherit_counter(sub, parent, parent_ctx,
                                            child, leader, child_ctx);
                if (IS_ERR(child_ctr))
                child_ctr = inherit_counter(sub, parent, parent_ctx,
                                            child, leader, child_ctx);
                if (IS_ERR(child_ctr))


@@ -4631,8 +4669,8 @@ void perf_counter_exit_task(struct task_struct *child)
        mutex_lock_nested(&child_ctx->mutex, SINGLE_DEPTH_NESTING);
 
 again:
        mutex_lock_nested(&child_ctx->mutex, SINGLE_DEPTH_NESTING);
 
 again:

-       list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
-                                list_entry)
+       list_for_each_entry_safe(child_counter, tmp, &child_ctx->group_list,
+                                group_entry)

                __perf_counter_exit_task(child_counter, child_ctx, child);
 
        /*
                __perf_counter_exit_task(child_counter, child_ctx, child);
 
        /*


@@ -4640,7 +4678,7 @@ again:
         * its siblings to the list, but we obtained 'tmp' before that which
         * will still point to the list head terminating the iteration.
         */
         * its siblings to the list, but we obtained 'tmp' before that which
         * will still point to the list head terminating the iteration.
         */

-       if (!list_empty(&child_ctx->counter_list))
+       if (!list_empty(&child_ctx->group_list))

                goto again;
 
        mutex_unlock(&child_ctx->mutex);
                goto again;
 
        mutex_unlock(&child_ctx->mutex);


@@ -4662,7 +4700,7 @@ void perf_counter_free_task(struct task_struct *task)
 
        mutex_lock(&ctx->mutex);
 again:
 
        mutex_lock(&ctx->mutex);
 again:

-       list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry) {
+       list_for_each_entry_safe(counter, tmp, &ctx->group_list, group_entry) {

                struct perf_counter *parent = counter->parent;
 
                if (WARN_ON_ONCE(!parent))
                struct perf_counter *parent = counter->parent;
 
                if (WARN_ON_ONCE(!parent))


@@ -4678,7 +4716,7 @@ again:
                free_counter(counter);
        }
 
                free_counter(counter);
        }
 

-       if (!list_empty(&ctx->counter_list))
+       if (!list_empty(&ctx->group_list))

                goto again;
 
        mutex_unlock(&ctx->mutex);
                goto again;
 
        mutex_unlock(&ctx->mutex);


@@ -4808,7 +4846,7 @@ static void __perf_counter_exit_cpu(void *info)
        struct perf_counter_context *ctx = &cpuctx->ctx;
        struct perf_counter *counter, *tmp;
 
        struct perf_counter_context *ctx = &cpuctx->ctx;
        struct perf_counter *counter, *tmp;
 

-       list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
+       list_for_each_entry_safe(counter, tmp, &ctx->group_list, group_entry)

                __perf_counter_remove_from_context(counter);
 }
 static void perf_counter_exit_cpu(int cpu)
                __perf_counter_remove_from_context(counter);
 }
 static void perf_counter_exit_cpu(int cpu)