projects
/
platform
/
adaptation
/
renesas_rcar
/
renesas_kernel.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Merge tag 'backport/v3.14.24-ltsi-rc1/pci-rcar-gen2-to-v3.16' into backport/v3.14...
[platform/adaptation/renesas_rcar/renesas_kernel.git]
/
kernel
/
smp.c
diff --git
a/kernel/smp.c
b/kernel/smp.c
index
bd9f940
..
ff87d44
100644
(file)
--- a/
kernel/smp.c
+++ b/
kernel/smp.c
@@
-23,17
+23,11
@@
enum {
struct call_function_data {
struct call_single_data __percpu *csd;
cpumask_var_t cpumask;
struct call_function_data {
struct call_single_data __percpu *csd;
cpumask_var_t cpumask;
- cpumask_var_t cpumask_ipi;
};
static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_function_data, cfd_data);
};
static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_function_data, cfd_data);
-struct call_single_queue {
- struct list_head list;
- raw_spinlock_t lock;
-};
-
-static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_queue, call_single_queue);
+static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);
static int
hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
static int
hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
@@
-47,14
+41,8
@@
hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
cpu_to_node(cpu)))
return notifier_from_errno(-ENOMEM);
if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
cpu_to_node(cpu)))
return notifier_from_errno(-ENOMEM);
- if (!zalloc_cpumask_var_node(&cfd->cpumask_ipi, GFP_KERNEL,
- cpu_to_node(cpu))) {
- free_cpumask_var(cfd->cpumask);
- return notifier_from_errno(-ENOMEM);
- }
cfd->csd = alloc_percpu(struct call_single_data);
if (!cfd->csd) {
cfd->csd = alloc_percpu(struct call_single_data);
if (!cfd->csd) {
- free_cpumask_var(cfd->cpumask_ipi);
free_cpumask_var(cfd->cpumask);
return notifier_from_errno(-ENOMEM);
}
free_cpumask_var(cfd->cpumask);
return notifier_from_errno(-ENOMEM);
}
@@
-67,7
+55,6
@@
hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
case CPU_DEAD:
case CPU_DEAD_FROZEN:
free_cpumask_var(cfd->cpumask);
case CPU_DEAD:
case CPU_DEAD_FROZEN:
free_cpumask_var(cfd->cpumask);
- free_cpumask_var(cfd->cpumask_ipi);
free_percpu(cfd->csd);
break;
#endif
free_percpu(cfd->csd);
break;
#endif
@@
-85,12
+72,8
@@
void __init call_function_init(void)
void *cpu = (void *)(long)smp_processor_id();
int i;
void *cpu = (void *)(long)smp_processor_id();
int i;
- for_each_possible_cpu(i) {
- struct call_single_queue *q = &per_cpu(call_single_queue, i);
-
- raw_spin_lock_init(&q->lock);
- INIT_LIST_HEAD(&q->list);
- }
+ for_each_possible_cpu(i)
+ init_llist_head(&per_cpu(call_single_queue, i));
hotplug_cfd(&hotplug_cfd_notifier, CPU_UP_PREPARE, cpu);
register_cpu_notifier(&hotplug_cfd_notifier);
hotplug_cfd(&hotplug_cfd_notifier, CPU_UP_PREPARE, cpu);
register_cpu_notifier(&hotplug_cfd_notifier);
@@
-141,18
+124,9
@@
static void csd_unlock(struct call_single_data *csd)
*/
static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
{
*/
static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
{
- struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
- unsigned long flags;
- int ipi;
-
if (wait)
csd->flags |= CSD_FLAG_WAIT;
if (wait)
csd->flags |= CSD_FLAG_WAIT;
- raw_spin_lock_irqsave(&dst->lock, flags);
- ipi = list_empty(&dst->list);
- list_add_tail(&csd->list, &dst->list);
- raw_spin_unlock_irqrestore(&dst->lock, flags);
-
/*
* The list addition should be visible before sending the IPI
* handler locks the list to pull the entry off it because of
/*
* The list addition should be visible before sending the IPI
* handler locks the list to pull the entry off it because of
@@
-164,7
+138,7
@@
static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
* locking and barrier primitives. Generic code isn't really
* equipped to do the right thing...
*/
* locking and barrier primitives. Generic code isn't really
* equipped to do the right thing...
*/
- if (
ipi
)
+ if (
llist_add(&csd->llist, &per_cpu(call_single_queue, cpu))
)
arch_send_call_function_single_ipi(cpu);
if (wait)
arch_send_call_function_single_ipi(cpu);
if (wait)
@@
-177,27
+151,26
@@
static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
*/
void generic_smp_call_function_single_interrupt(void)
{
*/
void generic_smp_call_function_single_interrupt(void)
{
- struct call_single_queue *q = &__get_cpu_var(call_single_queue);
- LIST_HEAD(list);
+ struct llist_node *entry, *next;
/*
* Shouldn't receive this interrupt on a cpu that is not yet online.
*/
WARN_ON_ONCE(!cpu_online(smp_processor_id()));
/*
* Shouldn't receive this interrupt on a cpu that is not yet online.
*/
WARN_ON_ONCE(!cpu_online(smp_processor_id()));
- raw_spin_lock(&q->lock);
- list_replace_init(&q->list, &list);
- raw_spin_unlock(&q->lock);
+ entry = llist_del_all(&__get_cpu_var(call_single_queue));
+ entry = llist_reverse_order(entry);
- while (
!list_empty(&list)
) {
+ while (
entry
) {
struct call_single_data *csd;
struct call_single_data *csd;
- csd = list_entry(list.next, struct call_single_data, list);
- list_del(&csd->list);
+ next = entry->next;
+ csd = llist_entry(entry, struct call_single_data, llist);
csd->func(csd->info);
csd->func(csd->info);
-
csd_unlock(csd);
csd_unlock(csd);
+
+ entry = next;
}
}
}
}
@@
-402,30
+375,17
@@
void smp_call_function_many(const struct cpumask *mask,
if (unlikely(!cpumask_weight(cfd->cpumask)))
return;
if (unlikely(!cpumask_weight(cfd->cpumask)))
return;
- /*
- * After we put an entry into the list, cfd->cpumask may be cleared
- * again when another CPU sends another IPI for a SMP function call, so
- * cfd->cpumask will be zero.
- */
- cpumask_copy(cfd->cpumask_ipi, cfd->cpumask);
-
for_each_cpu(cpu, cfd->cpumask) {
struct call_single_data *csd = per_cpu_ptr(cfd->csd, cpu);
for_each_cpu(cpu, cfd->cpumask) {
struct call_single_data *csd = per_cpu_ptr(cfd->csd, cpu);
- struct call_single_queue *dst =
- &per_cpu(call_single_queue, cpu);
- unsigned long flags;
csd_lock(csd);
csd->func = func;
csd->info = info;
csd_lock(csd);
csd->func = func;
csd->info = info;
-
- raw_spin_lock_irqsave(&dst->lock, flags);
- list_add_tail(&csd->list, &dst->list);
- raw_spin_unlock_irqrestore(&dst->lock, flags);
+ llist_add(&csd->llist, &per_cpu(call_single_queue, cpu));
}
/* Send a message to all CPUs in the map */
}
/* Send a message to all CPUs in the map */
- arch_send_call_function_ipi_mask(cfd->cpumask
_ipi
);
+ arch_send_call_function_ipi_mask(cfd->cpumask);
if (wait) {
for_each_cpu(cpu, cfd->cpumask) {
if (wait) {
for_each_cpu(cpu, cfd->cpumask) {
@@
-657,7
+617,7
@@
void on_each_cpu_cond(bool (*cond_func)(int cpu, void *info),
if (cond_func(cpu, info)) {
ret = smp_call_function_single(cpu, func,
info, wait);
if (cond_func(cpu, info)) {
ret = smp_call_function_single(cpu, func,
info, wait);
- WARN_ON_ONCE(
!
ret);
+ WARN_ON_ONCE(ret);
}
preempt_enable();
}
}
preempt_enable();
}