tick: Dynamically set broadcast irq affinity
[platform/adaptation/renesas_rcar/renesas_kernel.git] / kernel / time / tick-broadcast.c
1 /*
2  * linux/kernel/time/tick-broadcast.c
3  *
4  * This file contains functions which emulate a local clock-event
5  * device via a broadcast event source.
6  *
7  * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
8  * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
9  * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
10  *
11  * This code is licenced under the GPL version 2. For details see
12  * kernel-base/COPYING.
13  */
14 #include <linux/cpu.h>
15 #include <linux/err.h>
16 #include <linux/hrtimer.h>
17 #include <linux/interrupt.h>
18 #include <linux/percpu.h>
19 #include <linux/profile.h>
20 #include <linux/sched.h>
21 #include <linux/smp.h>
22
23 #include "tick-internal.h"
24
25 /*
26  * Broadcast support for broken x86 hardware, where the local apic
27  * timer stops in C3 state.
28  */
29
30 static struct tick_device tick_broadcast_device;
31 static cpumask_var_t tick_broadcast_mask;
32 static cpumask_var_t tmpmask;
33 static DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
34 static int tick_broadcast_force;
35
36 #ifdef CONFIG_TICK_ONESHOT
37 static void tick_broadcast_clear_oneshot(int cpu);
38 #else
39 static inline void tick_broadcast_clear_oneshot(int cpu) { }
40 #endif
41
42 /*
43  * Debugging: see timer_list.c
44  */
45 struct tick_device *tick_get_broadcast_device(void)
46 {
47         return &tick_broadcast_device;
48 }
49
50 struct cpumask *tick_get_broadcast_mask(void)
51 {
52         return tick_broadcast_mask;
53 }
54
55 /*
56  * Start the device in periodic mode
57  */
58 static void tick_broadcast_start_periodic(struct clock_event_device *bc)
59 {
60         if (bc)
61                 tick_setup_periodic(bc, 1);
62 }
63
64 /*
65  * Check, if the device can be utilized as broadcast device:
66  */
67 int tick_check_broadcast_device(struct clock_event_device *dev)
68 {
69         if ((tick_broadcast_device.evtdev &&
70              tick_broadcast_device.evtdev->rating >= dev->rating) ||
71              (dev->features & CLOCK_EVT_FEAT_C3STOP))
72                 return 0;
73
74         clockevents_exchange_device(tick_broadcast_device.evtdev, dev);
75         tick_broadcast_device.evtdev = dev;
76         if (!cpumask_empty(tick_broadcast_mask))
77                 tick_broadcast_start_periodic(dev);
78         return 1;
79 }
80
81 /*
82  * Check, if the device is the broadcast device
83  */
84 int tick_is_broadcast_device(struct clock_event_device *dev)
85 {
86         return (dev && tick_broadcast_device.evtdev == dev);
87 }
88
89 static void err_broadcast(const struct cpumask *mask)
90 {
91         pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");
92 }
93
94 static void tick_device_setup_broadcast_func(struct clock_event_device *dev)
95 {
96         if (!dev->broadcast)
97                 dev->broadcast = tick_broadcast;
98         if (!dev->broadcast) {
99                 pr_warn_once("%s depends on broadcast, but no broadcast function available\n",
100                              dev->name);
101                 dev->broadcast = err_broadcast;
102         }
103 }
104
105 /*
106  * Check, if the device is disfunctional and a place holder, which
107  * needs to be handled by the broadcast device.
108  */
109 int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
110 {
111         unsigned long flags;
112         int ret = 0;
113
114         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
115
116         /*
117          * Devices might be registered with both periodic and oneshot
118          * mode disabled. This signals, that the device needs to be
119          * operated from the broadcast device and is a placeholder for
120          * the cpu local device.
121          */
122         if (!tick_device_is_functional(dev)) {
123                 dev->event_handler = tick_handle_periodic;
124                 tick_device_setup_broadcast_func(dev);
125                 cpumask_set_cpu(cpu, tick_broadcast_mask);
126                 tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
127                 ret = 1;
128         } else {
129                 /*
130                  * When the new device is not affected by the stop
131                  * feature and the cpu is marked in the broadcast mask
132                  * then clear the broadcast bit.
133                  */
134                 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) {
135                         int cpu = smp_processor_id();
136                         cpumask_clear_cpu(cpu, tick_broadcast_mask);
137                         tick_broadcast_clear_oneshot(cpu);
138                 } else {
139                         tick_device_setup_broadcast_func(dev);
140                 }
141         }
142         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
143         return ret;
144 }
145
146 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
147 int tick_receive_broadcast(void)
148 {
149         struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
150         struct clock_event_device *evt = td->evtdev;
151
152         if (!evt)
153                 return -ENODEV;
154
155         if (!evt->event_handler)
156                 return -EINVAL;
157
158         evt->event_handler(evt);
159         return 0;
160 }
161 #endif
162
163 /*
164  * Broadcast the event to the cpus, which are set in the mask (mangled).
165  */
166 static void tick_do_broadcast(struct cpumask *mask)
167 {
168         int cpu = smp_processor_id();
169         struct tick_device *td;
170
171         /*
172          * Check, if the current cpu is in the mask
173          */
174         if (cpumask_test_cpu(cpu, mask)) {
175                 cpumask_clear_cpu(cpu, mask);
176                 td = &per_cpu(tick_cpu_device, cpu);
177                 td->evtdev->event_handler(td->evtdev);
178         }
179
180         if (!cpumask_empty(mask)) {
181                 /*
182                  * It might be necessary to actually check whether the devices
183                  * have different broadcast functions. For now, just use the
184                  * one of the first device. This works as long as we have this
185                  * misfeature only on x86 (lapic)
186                  */
187                 td = &per_cpu(tick_cpu_device, cpumask_first(mask));
188                 td->evtdev->broadcast(mask);
189         }
190 }
191
192 /*
193  * Periodic broadcast:
194  * - invoke the broadcast handlers
195  */
196 static void tick_do_periodic_broadcast(void)
197 {
198         raw_spin_lock(&tick_broadcast_lock);
199
200         cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask);
201         tick_do_broadcast(tmpmask);
202
203         raw_spin_unlock(&tick_broadcast_lock);
204 }
205
206 /*
207  * Event handler for periodic broadcast ticks
208  */
209 static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
210 {
211         ktime_t next;
212
213         tick_do_periodic_broadcast();
214
215         /*
216          * The device is in periodic mode. No reprogramming necessary:
217          */
218         if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
219                 return;
220
221         /*
222          * Setup the next period for devices, which do not have
223          * periodic mode. We read dev->next_event first and add to it
224          * when the event already expired. clockevents_program_event()
225          * sets dev->next_event only when the event is really
226          * programmed to the device.
227          */
228         for (next = dev->next_event; ;) {
229                 next = ktime_add(next, tick_period);
230
231                 if (!clockevents_program_event(dev, next, false))
232                         return;
233                 tick_do_periodic_broadcast();
234         }
235 }
236
237 /*
238  * Powerstate information: The system enters/leaves a state, where
239  * affected devices might stop
240  */
241 static void tick_do_broadcast_on_off(unsigned long *reason)
242 {
243         struct clock_event_device *bc, *dev;
244         struct tick_device *td;
245         unsigned long flags;
246         int cpu, bc_stopped;
247
248         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
249
250         cpu = smp_processor_id();
251         td = &per_cpu(tick_cpu_device, cpu);
252         dev = td->evtdev;
253         bc = tick_broadcast_device.evtdev;
254
255         /*
256          * Is the device not affected by the powerstate ?
257          */
258         if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
259                 goto out;
260
261         if (!tick_device_is_functional(dev))
262                 goto out;
263
264         bc_stopped = cpumask_empty(tick_broadcast_mask);
265
266         switch (*reason) {
267         case CLOCK_EVT_NOTIFY_BROADCAST_ON:
268         case CLOCK_EVT_NOTIFY_BROADCAST_FORCE:
269                 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) {
270                         if (tick_broadcast_device.mode ==
271                             TICKDEV_MODE_PERIODIC)
272                                 clockevents_shutdown(dev);
273                 }
274                 if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE)
275                         tick_broadcast_force = 1;
276                 break;
277         case CLOCK_EVT_NOTIFY_BROADCAST_OFF:
278                 if (!tick_broadcast_force &&
279                     cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) {
280                         if (tick_broadcast_device.mode ==
281                             TICKDEV_MODE_PERIODIC)
282                                 tick_setup_periodic(dev, 0);
283                 }
284                 break;
285         }
286
287         if (cpumask_empty(tick_broadcast_mask)) {
288                 if (!bc_stopped)
289                         clockevents_shutdown(bc);
290         } else if (bc_stopped) {
291                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
292                         tick_broadcast_start_periodic(bc);
293                 else
294                         tick_broadcast_setup_oneshot(bc);
295         }
296 out:
297         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
298 }
299
300 /*
301  * Powerstate information: The system enters/leaves a state, where
302  * affected devices might stop.
303  */
304 void tick_broadcast_on_off(unsigned long reason, int *oncpu)
305 {
306         if (!cpumask_test_cpu(*oncpu, cpu_online_mask))
307                 printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
308                        "offline CPU #%d\n", *oncpu);
309         else
310                 tick_do_broadcast_on_off(&reason);
311 }
312
313 /*
314  * Set the periodic handler depending on broadcast on/off
315  */
316 void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
317 {
318         if (!broadcast)
319                 dev->event_handler = tick_handle_periodic;
320         else
321                 dev->event_handler = tick_handle_periodic_broadcast;
322 }
323
324 /*
325  * Remove a CPU from broadcasting
326  */
327 void tick_shutdown_broadcast(unsigned int *cpup)
328 {
329         struct clock_event_device *bc;
330         unsigned long flags;
331         unsigned int cpu = *cpup;
332
333         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
334
335         bc = tick_broadcast_device.evtdev;
336         cpumask_clear_cpu(cpu, tick_broadcast_mask);
337
338         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
339                 if (bc && cpumask_empty(tick_broadcast_mask))
340                         clockevents_shutdown(bc);
341         }
342
343         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
344 }
345
346 void tick_suspend_broadcast(void)
347 {
348         struct clock_event_device *bc;
349         unsigned long flags;
350
351         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
352
353         bc = tick_broadcast_device.evtdev;
354         if (bc)
355                 clockevents_shutdown(bc);
356
357         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
358 }
359
360 int tick_resume_broadcast(void)
361 {
362         struct clock_event_device *bc;
363         unsigned long flags;
364         int broadcast = 0;
365
366         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
367
368         bc = tick_broadcast_device.evtdev;
369
370         if (bc) {
371                 clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME);
372
373                 switch (tick_broadcast_device.mode) {
374                 case TICKDEV_MODE_PERIODIC:
375                         if (!cpumask_empty(tick_broadcast_mask))
376                                 tick_broadcast_start_periodic(bc);
377                         broadcast = cpumask_test_cpu(smp_processor_id(),
378                                                      tick_broadcast_mask);
379                         break;
380                 case TICKDEV_MODE_ONESHOT:
381                         if (!cpumask_empty(tick_broadcast_mask))
382                                 broadcast = tick_resume_broadcast_oneshot(bc);
383                         break;
384                 }
385         }
386         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
387
388         return broadcast;
389 }
390
391
392 #ifdef CONFIG_TICK_ONESHOT
393
394 static cpumask_var_t tick_broadcast_oneshot_mask;
395
396 /*
397  * Exposed for debugging: see timer_list.c
398  */
399 struct cpumask *tick_get_broadcast_oneshot_mask(void)
400 {
401         return tick_broadcast_oneshot_mask;
402 }
403
404 /*
405  * Set broadcast interrupt affinity
406  */
407 static void tick_broadcast_set_affinity(struct clock_event_device *bc,
408                                         const struct cpumask *cpumask)
409 {
410         if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ))
411                 return;
412
413         if (cpumask_equal(bc->cpumask, cpumask))
414                 return;
415
416         bc->cpumask = cpumask;
417         irq_set_affinity(bc->irq, bc->cpumask);
418 }
419
420 static int tick_broadcast_set_event(struct clock_event_device *bc, int cpu,
421                                     ktime_t expires, int force)
422 {
423         int ret;
424
425         if (bc->mode != CLOCK_EVT_MODE_ONESHOT)
426                 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
427
428         ret = clockevents_program_event(bc, expires, force);
429         if (!ret)
430                 tick_broadcast_set_affinity(bc, cpumask_of(cpu));
431         return ret;
432 }
433
434 int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
435 {
436         clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
437         return 0;
438 }
439
440 /*
441  * Called from irq_enter() when idle was interrupted to reenable the
442  * per cpu device.
443  */
444 void tick_check_oneshot_broadcast(int cpu)
445 {
446         if (cpumask_test_cpu(cpu, tick_broadcast_oneshot_mask)) {
447                 struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
448
449                 clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT);
450         }
451 }
452
453 /*
454  * Handle oneshot mode broadcasting
455  */
456 static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
457 {
458         struct tick_device *td;
459         ktime_t now, next_event;
460         int cpu, next_cpu = 0;
461
462         raw_spin_lock(&tick_broadcast_lock);
463 again:
464         dev->next_event.tv64 = KTIME_MAX;
465         next_event.tv64 = KTIME_MAX;
466         cpumask_clear(tmpmask);
467         now = ktime_get();
468         /* Find all expired events */
469         for_each_cpu(cpu, tick_broadcast_oneshot_mask) {
470                 td = &per_cpu(tick_cpu_device, cpu);
471                 if (td->evtdev->next_event.tv64 <= now.tv64) {
472                         cpumask_set_cpu(cpu, tmpmask);
473                 } else if (td->evtdev->next_event.tv64 < next_event.tv64) {
474                         next_event.tv64 = td->evtdev->next_event.tv64;
475                         next_cpu = cpu;
476                 }
477         }
478
479         /*
480          * Wakeup the cpus which have an expired event.
481          */
482         tick_do_broadcast(tmpmask);
483
484         /*
485          * Two reasons for reprogram:
486          *
487          * - The global event did not expire any CPU local
488          * events. This happens in dyntick mode, as the maximum PIT
489          * delta is quite small.
490          *
491          * - There are pending events on sleeping CPUs which were not
492          * in the event mask
493          */
494         if (next_event.tv64 != KTIME_MAX) {
495                 /*
496                  * Rearm the broadcast device. If event expired,
497                  * repeat the above
498                  */
499                 if (tick_broadcast_set_event(dev, next_cpu, next_event, 0))
500                         goto again;
501         }
502         raw_spin_unlock(&tick_broadcast_lock);
503 }
504
505 /*
506  * Powerstate information: The system enters/leaves a state, where
507  * affected devices might stop
508  */
509 void tick_broadcast_oneshot_control(unsigned long reason)
510 {
511         struct clock_event_device *bc, *dev;
512         struct tick_device *td;
513         unsigned long flags;
514         int cpu;
515
516         /*
517          * Periodic mode does not care about the enter/exit of power
518          * states
519          */
520         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
521                 return;
522
523         /*
524          * We are called with preemtion disabled from the depth of the
525          * idle code, so we can't be moved away.
526          */
527         cpu = smp_processor_id();
528         td = &per_cpu(tick_cpu_device, cpu);
529         dev = td->evtdev;
530
531         if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
532                 return;
533
534         bc = tick_broadcast_device.evtdev;
535
536         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
537         if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
538                 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) {
539                         clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
540                         if (dev->next_event.tv64 < bc->next_event.tv64)
541                                 tick_broadcast_set_event(bc, cpu, dev->next_event, 1);
542                 }
543         } else {
544                 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) {
545                         clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
546                         if (dev->next_event.tv64 != KTIME_MAX)
547                                 tick_program_event(dev->next_event, 1);
548                 }
549         }
550         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
551 }
552
553 /*
554  * Reset the one shot broadcast for a cpu
555  *
556  * Called with tick_broadcast_lock held
557  */
558 static void tick_broadcast_clear_oneshot(int cpu)
559 {
560         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
561 }
562
563 static void tick_broadcast_init_next_event(struct cpumask *mask,
564                                            ktime_t expires)
565 {
566         struct tick_device *td;
567         int cpu;
568
569         for_each_cpu(cpu, mask) {
570                 td = &per_cpu(tick_cpu_device, cpu);
571                 if (td->evtdev)
572                         td->evtdev->next_event = expires;
573         }
574 }
575
576 /**
577  * tick_broadcast_setup_oneshot - setup the broadcast device
578  */
579 void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
580 {
581         int cpu = smp_processor_id();
582
583         /* Set it up only once ! */
584         if (bc->event_handler != tick_handle_oneshot_broadcast) {
585                 int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC;
586
587                 bc->event_handler = tick_handle_oneshot_broadcast;
588
589                 /* Take the do_timer update */
590                 tick_do_timer_cpu = cpu;
591
592                 /*
593                  * We must be careful here. There might be other CPUs
594                  * waiting for periodic broadcast. We need to set the
595                  * oneshot_mask bits for those and program the
596                  * broadcast device to fire.
597                  */
598                 cpumask_copy(tmpmask, tick_broadcast_mask);
599                 cpumask_clear_cpu(cpu, tmpmask);
600                 cpumask_or(tick_broadcast_oneshot_mask,
601                            tick_broadcast_oneshot_mask, tmpmask);
602
603                 if (was_periodic && !cpumask_empty(tmpmask)) {
604                         clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
605                         tick_broadcast_init_next_event(tmpmask,
606                                                        tick_next_period);
607                         tick_broadcast_set_event(bc, cpu, tick_next_period, 1);
608                 } else
609                         bc->next_event.tv64 = KTIME_MAX;
610         } else {
611                 /*
612                  * The first cpu which switches to oneshot mode sets
613                  * the bit for all other cpus which are in the general
614                  * (periodic) broadcast mask. So the bit is set and
615                  * would prevent the first broadcast enter after this
616                  * to program the bc device.
617                  */
618                 tick_broadcast_clear_oneshot(cpu);
619         }
620 }
621
622 /*
623  * Select oneshot operating mode for the broadcast device
624  */
625 void tick_broadcast_switch_to_oneshot(void)
626 {
627         struct clock_event_device *bc;
628         unsigned long flags;
629
630         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
631
632         tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
633         bc = tick_broadcast_device.evtdev;
634         if (bc)
635                 tick_broadcast_setup_oneshot(bc);
636
637         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
638 }
639
640
641 /*
642  * Remove a dead CPU from broadcasting
643  */
644 void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
645 {
646         unsigned long flags;
647         unsigned int cpu = *cpup;
648
649         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
650
651         /*
652          * Clear the broadcast mask flag for the dead cpu, but do not
653          * stop the broadcast device!
654          */
655         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
656
657         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
658 }
659
660 /*
661  * Check, whether the broadcast device is in one shot mode
662  */
663 int tick_broadcast_oneshot_active(void)
664 {
665         return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
666 }
667
668 /*
669  * Check whether the broadcast device supports oneshot.
670  */
671 bool tick_broadcast_oneshot_available(void)
672 {
673         struct clock_event_device *bc = tick_broadcast_device.evtdev;
674
675         return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
676 }
677
678 #endif
679
680 void __init tick_broadcast_init(void)
681 {
682         alloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT);
683         alloc_cpumask_var(&tmpmask, GFP_NOWAIT);
684 #ifdef CONFIG_TICK_ONESHOT
685         alloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT);
686 #endif
687 }