Merge tag 'scsi-misc' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi
[platform/kernel/linux-exynos.git] / drivers / clocksource / arm_arch_timer.c
1 /*
2  *  linux/drivers/clocksource/arm_arch_timer.c
3  *
4  *  Copyright (C) 2011 ARM Ltd.
5  *  All Rights Reserved
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License version 2 as
9  * published by the Free Software Foundation.
10  */
11
12 #define pr_fmt(fmt)     "arm_arch_timer: " fmt
13
14 #include <linux/init.h>
15 #include <linux/kernel.h>
16 #include <linux/device.h>
17 #include <linux/smp.h>
18 #include <linux/cpu.h>
19 #include <linux/cpu_pm.h>
20 #include <linux/clockchips.h>
21 #include <linux/clocksource.h>
22 #include <linux/interrupt.h>
23 #include <linux/of_irq.h>
24 #include <linux/of_address.h>
25 #include <linux/io.h>
26 #include <linux/slab.h>
27 #include <linux/sched/clock.h>
28 #include <linux/sched_clock.h>
29 #include <linux/acpi.h>
30
31 #include <asm/arch_timer.h>
32 #include <asm/virt.h>
33
34 #include <clocksource/arm_arch_timer.h>
35
36 #undef pr_fmt
37 #define pr_fmt(fmt) "arch_timer: " fmt
38
39 #define CNTTIDR         0x08
40 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
41
42 #define CNTACR(n)       (0x40 + ((n) * 4))
43 #define CNTACR_RPCT     BIT(0)
44 #define CNTACR_RVCT     BIT(1)
45 #define CNTACR_RFRQ     BIT(2)
46 #define CNTACR_RVOFF    BIT(3)
47 #define CNTACR_RWVT     BIT(4)
48 #define CNTACR_RWPT     BIT(5)
49
50 #define CNTVCT_LO       0x08
51 #define CNTVCT_HI       0x0c
52 #define CNTFRQ          0x10
53 #define CNTP_TVAL       0x28
54 #define CNTP_CTL        0x2c
55 #define CNTV_TVAL       0x38
56 #define CNTV_CTL        0x3c
57
58 static unsigned arch_timers_present __initdata;
59
60 static void __iomem *arch_counter_base;
61
62 struct arch_timer {
63         void __iomem *base;
64         struct clock_event_device evt;
65 };
66
67 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
68
69 static u32 arch_timer_rate;
70 static int arch_timer_ppi[ARCH_TIMER_MAX_TIMER_PPI];
71
72 static struct clock_event_device __percpu *arch_timer_evt;
73
74 static enum arch_timer_ppi_nr arch_timer_uses_ppi = ARCH_TIMER_VIRT_PPI;
75 static bool arch_timer_c3stop;
76 static bool arch_timer_mem_use_virtual;
77 static bool arch_counter_suspend_stop;
78 static bool vdso_default = true;
79
80 static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
81
82 static int __init early_evtstrm_cfg(char *buf)
83 {
84         return strtobool(buf, &evtstrm_enable);
85 }
86 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
87
88 /*
89  * Architected system timer support.
90  */
91
92 static __always_inline
93 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
94                           struct clock_event_device *clk)
95 {
96         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
97                 struct arch_timer *timer = to_arch_timer(clk);
98                 switch (reg) {
99                 case ARCH_TIMER_REG_CTRL:
100                         writel_relaxed(val, timer->base + CNTP_CTL);
101                         break;
102                 case ARCH_TIMER_REG_TVAL:
103                         writel_relaxed(val, timer->base + CNTP_TVAL);
104                         break;
105                 }
106         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
107                 struct arch_timer *timer = to_arch_timer(clk);
108                 switch (reg) {
109                 case ARCH_TIMER_REG_CTRL:
110                         writel_relaxed(val, timer->base + CNTV_CTL);
111                         break;
112                 case ARCH_TIMER_REG_TVAL:
113                         writel_relaxed(val, timer->base + CNTV_TVAL);
114                         break;
115                 }
116         } else {
117                 arch_timer_reg_write_cp15(access, reg, val);
118         }
119 }
120
121 static __always_inline
122 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
123                         struct clock_event_device *clk)
124 {
125         u32 val;
126
127         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
128                 struct arch_timer *timer = to_arch_timer(clk);
129                 switch (reg) {
130                 case ARCH_TIMER_REG_CTRL:
131                         val = readl_relaxed(timer->base + CNTP_CTL);
132                         break;
133                 case ARCH_TIMER_REG_TVAL:
134                         val = readl_relaxed(timer->base + CNTP_TVAL);
135                         break;
136                 }
137         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
138                 struct arch_timer *timer = to_arch_timer(clk);
139                 switch (reg) {
140                 case ARCH_TIMER_REG_CTRL:
141                         val = readl_relaxed(timer->base + CNTV_CTL);
142                         break;
143                 case ARCH_TIMER_REG_TVAL:
144                         val = readl_relaxed(timer->base + CNTV_TVAL);
145                         break;
146                 }
147         } else {
148                 val = arch_timer_reg_read_cp15(access, reg);
149         }
150
151         return val;
152 }
153
154 /*
155  * Default to cp15 based access because arm64 uses this function for
156  * sched_clock() before DT is probed and the cp15 method is guaranteed
157  * to exist on arm64. arm doesn't use this before DT is probed so even
158  * if we don't have the cp15 accessors we won't have a problem.
159  */
160 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
161
162 static u64 arch_counter_read(struct clocksource *cs)
163 {
164         return arch_timer_read_counter();
165 }
166
167 static u64 arch_counter_read_cc(const struct cyclecounter *cc)
168 {
169         return arch_timer_read_counter();
170 }
171
172 static struct clocksource clocksource_counter = {
173         .name   = "arch_sys_counter",
174         .rating = 400,
175         .read   = arch_counter_read,
176         .mask   = CLOCKSOURCE_MASK(56),
177         .flags  = CLOCK_SOURCE_IS_CONTINUOUS,
178 };
179
180 static struct cyclecounter cyclecounter __ro_after_init = {
181         .read   = arch_counter_read_cc,
182         .mask   = CLOCKSOURCE_MASK(56),
183 };
184
185 struct ate_acpi_oem_info {
186         char oem_id[ACPI_OEM_ID_SIZE + 1];
187         char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
188         u32 oem_revision;
189 };
190
191 #ifdef CONFIG_FSL_ERRATUM_A008585
192 /*
193  * The number of retries is an arbitrary value well beyond the highest number
194  * of iterations the loop has been observed to take.
195  */
196 #define __fsl_a008585_read_reg(reg) ({                  \
197         u64 _old, _new;                                 \
198         int _retries = 200;                             \
199                                                         \
200         do {                                            \
201                 _old = read_sysreg(reg);                \
202                 _new = read_sysreg(reg);                \
203                 _retries--;                             \
204         } while (unlikely(_old != _new) && _retries);   \
205                                                         \
206         WARN_ON_ONCE(!_retries);                        \
207         _new;                                           \
208 })
209
210 static u32 notrace fsl_a008585_read_cntp_tval_el0(void)
211 {
212         return __fsl_a008585_read_reg(cntp_tval_el0);
213 }
214
215 static u32 notrace fsl_a008585_read_cntv_tval_el0(void)
216 {
217         return __fsl_a008585_read_reg(cntv_tval_el0);
218 }
219
220 static u64 notrace fsl_a008585_read_cntvct_el0(void)
221 {
222         return __fsl_a008585_read_reg(cntvct_el0);
223 }
224 #endif
225
226 #ifdef CONFIG_HISILICON_ERRATUM_161010101
227 /*
228  * Verify whether the value of the second read is larger than the first by
229  * less than 32 is the only way to confirm the value is correct, so clear the
230  * lower 5 bits to check whether the difference is greater than 32 or not.
231  * Theoretically the erratum should not occur more than twice in succession
232  * when reading the system counter, but it is possible that some interrupts
233  * may lead to more than twice read errors, triggering the warning, so setting
234  * the number of retries far beyond the number of iterations the loop has been
235  * observed to take.
236  */
237 #define __hisi_161010101_read_reg(reg) ({                               \
238         u64 _old, _new;                                         \
239         int _retries = 50;                                      \
240                                                                 \
241         do {                                                    \
242                 _old = read_sysreg(reg);                        \
243                 _new = read_sysreg(reg);                        \
244                 _retries--;                                     \
245         } while (unlikely((_new - _old) >> 5) && _retries);     \
246                                                                 \
247         WARN_ON_ONCE(!_retries);                                \
248         _new;                                                   \
249 })
250
251 static u32 notrace hisi_161010101_read_cntp_tval_el0(void)
252 {
253         return __hisi_161010101_read_reg(cntp_tval_el0);
254 }
255
256 static u32 notrace hisi_161010101_read_cntv_tval_el0(void)
257 {
258         return __hisi_161010101_read_reg(cntv_tval_el0);
259 }
260
261 static u64 notrace hisi_161010101_read_cntvct_el0(void)
262 {
263         return __hisi_161010101_read_reg(cntvct_el0);
264 }
265
266 static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {
267         /*
268          * Note that trailing spaces are required to properly match
269          * the OEM table information.
270          */
271         {
272                 .oem_id         = "HISI  ",
273                 .oem_table_id   = "HIP05   ",
274                 .oem_revision   = 0,
275         },
276         {
277                 .oem_id         = "HISI  ",
278                 .oem_table_id   = "HIP06   ",
279                 .oem_revision   = 0,
280         },
281         {
282                 .oem_id         = "HISI  ",
283                 .oem_table_id   = "HIP07   ",
284                 .oem_revision   = 0,
285         },
286         { /* Sentinel indicating the end of the OEM array */ },
287 };
288 #endif
289
290 #ifdef CONFIG_ARM64_ERRATUM_858921
291 static u64 notrace arm64_858921_read_cntvct_el0(void)
292 {
293         u64 old, new;
294
295         old = read_sysreg(cntvct_el0);
296         new = read_sysreg(cntvct_el0);
297         return (((old ^ new) >> 32) & 1) ? old : new;
298 }
299 #endif
300
301 #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
302 DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *,
303                timer_unstable_counter_workaround);
304 EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround);
305
306 DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);
307 EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled);
308
309 static void erratum_set_next_event_tval_generic(const int access, unsigned long evt,
310                                                 struct clock_event_device *clk)
311 {
312         unsigned long ctrl;
313         u64 cval = evt + arch_counter_get_cntvct();
314
315         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
316         ctrl |= ARCH_TIMER_CTRL_ENABLE;
317         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
318
319         if (access == ARCH_TIMER_PHYS_ACCESS)
320                 write_sysreg(cval, cntp_cval_el0);
321         else
322                 write_sysreg(cval, cntv_cval_el0);
323
324         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
325 }
326
327 static __maybe_unused int erratum_set_next_event_tval_virt(unsigned long evt,
328                                             struct clock_event_device *clk)
329 {
330         erratum_set_next_event_tval_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);
331         return 0;
332 }
333
334 static __maybe_unused int erratum_set_next_event_tval_phys(unsigned long evt,
335                                             struct clock_event_device *clk)
336 {
337         erratum_set_next_event_tval_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);
338         return 0;
339 }
340
341 static const struct arch_timer_erratum_workaround ool_workarounds[] = {
342 #ifdef CONFIG_FSL_ERRATUM_A008585
343         {
344                 .match_type = ate_match_dt,
345                 .id = "fsl,erratum-a008585",
346                 .desc = "Freescale erratum a005858",
347                 .read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0,
348                 .read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0,
349                 .read_cntvct_el0 = fsl_a008585_read_cntvct_el0,
350                 .set_next_event_phys = erratum_set_next_event_tval_phys,
351                 .set_next_event_virt = erratum_set_next_event_tval_virt,
352         },
353 #endif
354 #ifdef CONFIG_HISILICON_ERRATUM_161010101
355         {
356                 .match_type = ate_match_dt,
357                 .id = "hisilicon,erratum-161010101",
358                 .desc = "HiSilicon erratum 161010101",
359                 .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,
360                 .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,
361                 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
362                 .set_next_event_phys = erratum_set_next_event_tval_phys,
363                 .set_next_event_virt = erratum_set_next_event_tval_virt,
364         },
365         {
366                 .match_type = ate_match_acpi_oem_info,
367                 .id = hisi_161010101_oem_info,
368                 .desc = "HiSilicon erratum 161010101",
369                 .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,
370                 .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,
371                 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
372                 .set_next_event_phys = erratum_set_next_event_tval_phys,
373                 .set_next_event_virt = erratum_set_next_event_tval_virt,
374         },
375 #endif
376 #ifdef CONFIG_ARM64_ERRATUM_858921
377         {
378                 .match_type = ate_match_local_cap_id,
379                 .id = (void *)ARM64_WORKAROUND_858921,
380                 .desc = "ARM erratum 858921",
381                 .read_cntvct_el0 = arm64_858921_read_cntvct_el0,
382         },
383 #endif
384 };
385
386 typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,
387                                const void *);
388
389 static
390 bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa,
391                                  const void *arg)
392 {
393         const struct device_node *np = arg;
394
395         return of_property_read_bool(np, wa->id);
396 }
397
398 static
399 bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa,
400                                         const void *arg)
401 {
402         return this_cpu_has_cap((uintptr_t)wa->id);
403 }
404
405
406 static
407 bool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa,
408                                        const void *arg)
409 {
410         static const struct ate_acpi_oem_info empty_oem_info = {};
411         const struct ate_acpi_oem_info *info = wa->id;
412         const struct acpi_table_header *table = arg;
413
414         /* Iterate over the ACPI OEM info array, looking for a match */
415         while (memcmp(info, &empty_oem_info, sizeof(*info))) {
416                 if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) &&
417                     !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
418                     info->oem_revision == table->oem_revision)
419                         return true;
420
421                 info++;
422         }
423
424         return false;
425 }
426
427 static const struct arch_timer_erratum_workaround *
428 arch_timer_iterate_errata(enum arch_timer_erratum_match_type type,
429                           ate_match_fn_t match_fn,
430                           void *arg)
431 {
432         int i;
433
434         for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) {
435                 if (ool_workarounds[i].match_type != type)
436                         continue;
437
438                 if (match_fn(&ool_workarounds[i], arg))
439                         return &ool_workarounds[i];
440         }
441
442         return NULL;
443 }
444
445 static
446 void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa,
447                                   bool local)
448 {
449         int i;
450
451         if (local) {
452                 __this_cpu_write(timer_unstable_counter_workaround, wa);
453         } else {
454                 for_each_possible_cpu(i)
455                         per_cpu(timer_unstable_counter_workaround, i) = wa;
456         }
457
458         /*
459          * Use the locked version, as we're called from the CPU
460          * hotplug framework. Otherwise, we end-up in deadlock-land.
461          */
462         static_branch_enable_cpuslocked(&arch_timer_read_ool_enabled);
463
464         /*
465          * Don't use the vdso fastpath if errata require using the
466          * out-of-line counter accessor. We may change our mind pretty
467          * late in the game (with a per-CPU erratum, for example), so
468          * change both the default value and the vdso itself.
469          */
470         if (wa->read_cntvct_el0) {
471                 clocksource_counter.archdata.vdso_direct = false;
472                 vdso_default = false;
473         }
474 }
475
476 static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,
477                                             void *arg)
478 {
479         const struct arch_timer_erratum_workaround *wa;
480         ate_match_fn_t match_fn = NULL;
481         bool local = false;
482
483         switch (type) {
484         case ate_match_dt:
485                 match_fn = arch_timer_check_dt_erratum;
486                 break;
487         case ate_match_local_cap_id:
488                 match_fn = arch_timer_check_local_cap_erratum;
489                 local = true;
490                 break;
491         case ate_match_acpi_oem_info:
492                 match_fn = arch_timer_check_acpi_oem_erratum;
493                 break;
494         default:
495                 WARN_ON(1);
496                 return;
497         }
498
499         wa = arch_timer_iterate_errata(type, match_fn, arg);
500         if (!wa)
501                 return;
502
503         if (needs_unstable_timer_counter_workaround()) {
504                 const struct arch_timer_erratum_workaround *__wa;
505                 __wa = __this_cpu_read(timer_unstable_counter_workaround);
506                 if (__wa && wa != __wa)
507                         pr_warn("Can't enable workaround for %s (clashes with %s\n)",
508                                 wa->desc, __wa->desc);
509
510                 if (__wa)
511                         return;
512         }
513
514         arch_timer_enable_workaround(wa, local);
515         pr_info("Enabling %s workaround for %s\n",
516                 local ? "local" : "global", wa->desc);
517 }
518
519 #define erratum_handler(fn, r, ...)                                     \
520 ({                                                                      \
521         bool __val;                                                     \
522         if (needs_unstable_timer_counter_workaround()) {                \
523                 const struct arch_timer_erratum_workaround *__wa;       \
524                 __wa = __this_cpu_read(timer_unstable_counter_workaround); \
525                 if (__wa && __wa->fn) {                                 \
526                         r = __wa->fn(__VA_ARGS__);                      \
527                         __val = true;                                   \
528                 } else {                                                \
529                         __val = false;                                  \
530                 }                                                       \
531         } else {                                                        \
532                 __val = false;                                          \
533         }                                                               \
534         __val;                                                          \
535 })
536
537 static bool arch_timer_this_cpu_has_cntvct_wa(void)
538 {
539         const struct arch_timer_erratum_workaround *wa;
540
541         wa = __this_cpu_read(timer_unstable_counter_workaround);
542         return wa && wa->read_cntvct_el0;
543 }
544 #else
545 #define arch_timer_check_ool_workaround(t,a)            do { } while(0)
546 #define erratum_set_next_event_tval_virt(...)           ({BUG(); 0;})
547 #define erratum_set_next_event_tval_phys(...)           ({BUG(); 0;})
548 #define erratum_handler(fn, r, ...)                     ({false;})
549 #define arch_timer_this_cpu_has_cntvct_wa()             ({false;})
550 #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
551
552 static __always_inline irqreturn_t timer_handler(const int access,
553                                         struct clock_event_device *evt)
554 {
555         unsigned long ctrl;
556
557         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
558         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
559                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
560                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
561                 evt->event_handler(evt);
562                 return IRQ_HANDLED;
563         }
564
565         return IRQ_NONE;
566 }
567
568 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
569 {
570         struct clock_event_device *evt = dev_id;
571
572         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
573 }
574
575 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
576 {
577         struct clock_event_device *evt = dev_id;
578
579         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
580 }
581
582 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
583 {
584         struct clock_event_device *evt = dev_id;
585
586         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
587 }
588
589 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
590 {
591         struct clock_event_device *evt = dev_id;
592
593         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
594 }
595
596 static __always_inline int timer_shutdown(const int access,
597                                           struct clock_event_device *clk)
598 {
599         unsigned long ctrl;
600
601         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
602         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
603         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
604
605         return 0;
606 }
607
608 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
609 {
610         return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
611 }
612
613 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
614 {
615         return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
616 }
617
618 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
619 {
620         return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
621 }
622
623 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
624 {
625         return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
626 }
627
628 static __always_inline void set_next_event(const int access, unsigned long evt,
629                                            struct clock_event_device *clk)
630 {
631         unsigned long ctrl;
632         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
633         ctrl |= ARCH_TIMER_CTRL_ENABLE;
634         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
635         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
636         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
637 }
638
639 static int arch_timer_set_next_event_virt(unsigned long evt,
640                                           struct clock_event_device *clk)
641 {
642         int ret;
643
644         if (erratum_handler(set_next_event_virt, ret, evt, clk))
645                 return ret;
646
647         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
648         return 0;
649 }
650
651 static int arch_timer_set_next_event_phys(unsigned long evt,
652                                           struct clock_event_device *clk)
653 {
654         int ret;
655
656         if (erratum_handler(set_next_event_phys, ret, evt, clk))
657                 return ret;
658
659         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
660         return 0;
661 }
662
663 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
664                                               struct clock_event_device *clk)
665 {
666         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
667         return 0;
668 }
669
670 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
671                                               struct clock_event_device *clk)
672 {
673         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
674         return 0;
675 }
676
677 static void __arch_timer_setup(unsigned type,
678                                struct clock_event_device *clk)
679 {
680         clk->features = CLOCK_EVT_FEAT_ONESHOT;
681
682         if (type == ARCH_TIMER_TYPE_CP15) {
683                 if (arch_timer_c3stop)
684                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
685                 clk->name = "arch_sys_timer";
686                 clk->rating = 450;
687                 clk->cpumask = cpumask_of(smp_processor_id());
688                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
689                 switch (arch_timer_uses_ppi) {
690                 case ARCH_TIMER_VIRT_PPI:
691                         clk->set_state_shutdown = arch_timer_shutdown_virt;
692                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
693                         clk->set_next_event = arch_timer_set_next_event_virt;
694                         break;
695                 case ARCH_TIMER_PHYS_SECURE_PPI:
696                 case ARCH_TIMER_PHYS_NONSECURE_PPI:
697                 case ARCH_TIMER_HYP_PPI:
698                         clk->set_state_shutdown = arch_timer_shutdown_phys;
699                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
700                         clk->set_next_event = arch_timer_set_next_event_phys;
701                         break;
702                 default:
703                         BUG();
704                 }
705
706                 arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);
707         } else {
708                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
709                 clk->name = "arch_mem_timer";
710                 clk->rating = 400;
711                 clk->cpumask = cpu_all_mask;
712                 if (arch_timer_mem_use_virtual) {
713                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
714                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
715                         clk->set_next_event =
716                                 arch_timer_set_next_event_virt_mem;
717                 } else {
718                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
719                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
720                         clk->set_next_event =
721                                 arch_timer_set_next_event_phys_mem;
722                 }
723         }
724
725         clk->set_state_shutdown(clk);
726
727         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
728 }
729
730 static void arch_timer_evtstrm_enable(int divider)
731 {
732         u32 cntkctl = arch_timer_get_cntkctl();
733
734         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
735         /* Set the divider and enable virtual event stream */
736         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
737                         | ARCH_TIMER_VIRT_EVT_EN;
738         arch_timer_set_cntkctl(cntkctl);
739         elf_hwcap |= HWCAP_EVTSTRM;
740 #ifdef CONFIG_COMPAT
741         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
742 #endif
743 }
744
745 static void arch_timer_configure_evtstream(void)
746 {
747         int evt_stream_div, pos;
748
749         /* Find the closest power of two to the divisor */
750         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
751         pos = fls(evt_stream_div);
752         if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
753                 pos--;
754         /* enable event stream */
755         arch_timer_evtstrm_enable(min(pos, 15));
756 }
757
758 static void arch_counter_set_user_access(void)
759 {
760         u32 cntkctl = arch_timer_get_cntkctl();
761
762         /* Disable user access to the timers and both counters */
763         /* Also disable virtual event stream */
764         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
765                         | ARCH_TIMER_USR_VT_ACCESS_EN
766                         | ARCH_TIMER_USR_VCT_ACCESS_EN
767                         | ARCH_TIMER_VIRT_EVT_EN
768                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
769
770         /*
771          * Enable user access to the virtual counter if it doesn't
772          * need to be workaround. The vdso may have been already
773          * disabled though.
774          */
775         if (arch_timer_this_cpu_has_cntvct_wa())
776                 pr_info("CPU%d: Trapping CNTVCT access\n", smp_processor_id());
777         else
778                 cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
779
780         arch_timer_set_cntkctl(cntkctl);
781 }
782
783 static bool arch_timer_has_nonsecure_ppi(void)
784 {
785         return (arch_timer_uses_ppi == ARCH_TIMER_PHYS_SECURE_PPI &&
786                 arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
787 }
788
789 static u32 check_ppi_trigger(int irq)
790 {
791         u32 flags = irq_get_trigger_type(irq);
792
793         if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
794                 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
795                 pr_warn("WARNING: Please fix your firmware\n");
796                 flags = IRQF_TRIGGER_LOW;
797         }
798
799         return flags;
800 }
801
802 static int arch_timer_starting_cpu(unsigned int cpu)
803 {
804         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
805         u32 flags;
806
807         __arch_timer_setup(ARCH_TIMER_TYPE_CP15, clk);
808
809         flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
810         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
811
812         if (arch_timer_has_nonsecure_ppi()) {
813                 flags = check_ppi_trigger(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
814                 enable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
815                                   flags);
816         }
817
818         arch_counter_set_user_access();
819         if (evtstrm_enable)
820                 arch_timer_configure_evtstream();
821
822         return 0;
823 }
824
825 /*
826  * For historical reasons, when probing with DT we use whichever (non-zero)
827  * rate was probed first, and don't verify that others match. If the first node
828  * probed has a clock-frequency property, this overrides the HW register.
829  */
830 static void arch_timer_of_configure_rate(u32 rate, struct device_node *np)
831 {
832         /* Who has more than one independent system counter? */
833         if (arch_timer_rate)
834                 return;
835
836         if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate))
837                 arch_timer_rate = rate;
838
839         /* Check the timer frequency. */
840         if (arch_timer_rate == 0)
841                 pr_warn("frequency not available\n");
842 }
843
844 static void arch_timer_banner(unsigned type)
845 {
846         pr_info("%s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
847                 type & ARCH_TIMER_TYPE_CP15 ? "cp15" : "",
848                 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ?
849                         " and " : "",
850                 type & ARCH_TIMER_TYPE_MEM ? "mmio" : "",
851                 (unsigned long)arch_timer_rate / 1000000,
852                 (unsigned long)(arch_timer_rate / 10000) % 100,
853                 type & ARCH_TIMER_TYPE_CP15 ?
854                         (arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) ? "virt" : "phys" :
855                         "",
856                 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? "/" : "",
857                 type & ARCH_TIMER_TYPE_MEM ?
858                         arch_timer_mem_use_virtual ? "virt" : "phys" :
859                         "");
860 }
861
862 u32 arch_timer_get_rate(void)
863 {
864         return arch_timer_rate;
865 }
866
867 static u64 arch_counter_get_cntvct_mem(void)
868 {
869         u32 vct_lo, vct_hi, tmp_hi;
870
871         do {
872                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
873                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
874                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
875         } while (vct_hi != tmp_hi);
876
877         return ((u64) vct_hi << 32) | vct_lo;
878 }
879
880 static struct arch_timer_kvm_info arch_timer_kvm_info;
881
882 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
883 {
884         return &arch_timer_kvm_info;
885 }
886
887 static void __init arch_counter_register(unsigned type)
888 {
889         u64 start_count;
890
891         /* Register the CP15 based counter if we have one */
892         if (type & ARCH_TIMER_TYPE_CP15) {
893                 if (IS_ENABLED(CONFIG_ARM64) ||
894                     arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI)
895                         arch_timer_read_counter = arch_counter_get_cntvct;
896                 else
897                         arch_timer_read_counter = arch_counter_get_cntpct;
898
899                 clocksource_counter.archdata.vdso_direct = vdso_default;
900         } else {
901                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
902         }
903
904         if (!arch_counter_suspend_stop)
905                 clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
906         start_count = arch_timer_read_counter();
907         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
908         cyclecounter.mult = clocksource_counter.mult;
909         cyclecounter.shift = clocksource_counter.shift;
910         timecounter_init(&arch_timer_kvm_info.timecounter,
911                          &cyclecounter, start_count);
912
913         /* 56 bits minimum, so we assume worst case rollover */
914         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
915 }
916
917 static void arch_timer_stop(struct clock_event_device *clk)
918 {
919         pr_debug("disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id());
920
921         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
922         if (arch_timer_has_nonsecure_ppi())
923                 disable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
924
925         clk->set_state_shutdown(clk);
926 }
927
928 static int arch_timer_dying_cpu(unsigned int cpu)
929 {
930         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
931
932         arch_timer_stop(clk);
933         return 0;
934 }
935
936 #ifdef CONFIG_CPU_PM
937 static DEFINE_PER_CPU(unsigned long, saved_cntkctl);
938 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
939                                     unsigned long action, void *hcpu)
940 {
941         if (action == CPU_PM_ENTER)
942                 __this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl());
943         else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
944                 arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
945         return NOTIFY_OK;
946 }
947
948 static struct notifier_block arch_timer_cpu_pm_notifier = {
949         .notifier_call = arch_timer_cpu_pm_notify,
950 };
951
952 static int __init arch_timer_cpu_pm_init(void)
953 {
954         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
955 }
956
957 static void __init arch_timer_cpu_pm_deinit(void)
958 {
959         WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
960 }
961
962 #else
963 static int __init arch_timer_cpu_pm_init(void)
964 {
965         return 0;
966 }
967
968 static void __init arch_timer_cpu_pm_deinit(void)
969 {
970 }
971 #endif
972
973 static int __init arch_timer_register(void)
974 {
975         int err;
976         int ppi;
977
978         arch_timer_evt = alloc_percpu(struct clock_event_device);
979         if (!arch_timer_evt) {
980                 err = -ENOMEM;
981                 goto out;
982         }
983
984         ppi = arch_timer_ppi[arch_timer_uses_ppi];
985         switch (arch_timer_uses_ppi) {
986         case ARCH_TIMER_VIRT_PPI:
987                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
988                                          "arch_timer", arch_timer_evt);
989                 break;
990         case ARCH_TIMER_PHYS_SECURE_PPI:
991         case ARCH_TIMER_PHYS_NONSECURE_PPI:
992                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
993                                          "arch_timer", arch_timer_evt);
994                 if (!err && arch_timer_has_nonsecure_ppi()) {
995                         ppi = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
996                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
997                                                  "arch_timer", arch_timer_evt);
998                         if (err)
999                                 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_SECURE_PPI],
1000                                                 arch_timer_evt);
1001                 }
1002                 break;
1003         case ARCH_TIMER_HYP_PPI:
1004                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
1005                                          "arch_timer", arch_timer_evt);
1006                 break;
1007         default:
1008                 BUG();
1009         }
1010
1011         if (err) {
1012                 pr_err("can't register interrupt %d (%d)\n", ppi, err);
1013                 goto out_free;
1014         }
1015
1016         err = arch_timer_cpu_pm_init();
1017         if (err)
1018                 goto out_unreg_notify;
1019
1020
1021         /* Register and immediately configure the timer on the boot CPU */
1022         err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
1023                                 "clockevents/arm/arch_timer:starting",
1024                                 arch_timer_starting_cpu, arch_timer_dying_cpu);
1025         if (err)
1026                 goto out_unreg_cpupm;
1027         return 0;
1028
1029 out_unreg_cpupm:
1030         arch_timer_cpu_pm_deinit();
1031
1032 out_unreg_notify:
1033         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
1034         if (arch_timer_has_nonsecure_ppi())
1035                 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
1036                                 arch_timer_evt);
1037
1038 out_free:
1039         free_percpu(arch_timer_evt);
1040 out:
1041         return err;
1042 }
1043
1044 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
1045 {
1046         int ret;
1047         irq_handler_t func;
1048         struct arch_timer *t;
1049
1050         t = kzalloc(sizeof(*t), GFP_KERNEL);
1051         if (!t)
1052                 return -ENOMEM;
1053
1054         t->base = base;
1055         t->evt.irq = irq;
1056         __arch_timer_setup(ARCH_TIMER_TYPE_MEM, &t->evt);
1057
1058         if (arch_timer_mem_use_virtual)
1059                 func = arch_timer_handler_virt_mem;
1060         else
1061                 func = arch_timer_handler_phys_mem;
1062
1063         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
1064         if (ret) {
1065                 pr_err("Failed to request mem timer irq\n");
1066                 kfree(t);
1067         }
1068
1069         return ret;
1070 }
1071
1072 static const struct of_device_id arch_timer_of_match[] __initconst = {
1073         { .compatible   = "arm,armv7-timer",    },
1074         { .compatible   = "arm,armv8-timer",    },
1075         {},
1076 };
1077
1078 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
1079         { .compatible   = "arm,armv7-timer-mem", },
1080         {},
1081 };
1082
1083 static bool __init arch_timer_needs_of_probing(void)
1084 {
1085         struct device_node *dn;
1086         bool needs_probing = false;
1087         unsigned int mask = ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM;
1088
1089         /* We have two timers, and both device-tree nodes are probed. */
1090         if ((arch_timers_present & mask) == mask)
1091                 return false;
1092
1093         /*
1094          * Only one type of timer is probed,
1095          * check if we have another type of timer node in device-tree.
1096          */
1097         if (arch_timers_present & ARCH_TIMER_TYPE_CP15)
1098                 dn = of_find_matching_node(NULL, arch_timer_mem_of_match);
1099         else
1100                 dn = of_find_matching_node(NULL, arch_timer_of_match);
1101
1102         if (dn && of_device_is_available(dn))
1103                 needs_probing = true;
1104
1105         of_node_put(dn);
1106
1107         return needs_probing;
1108 }
1109
1110 static int __init arch_timer_common_init(void)
1111 {
1112         arch_timer_banner(arch_timers_present);
1113         arch_counter_register(arch_timers_present);
1114         return arch_timer_arch_init();
1115 }
1116
1117 /**
1118  * arch_timer_select_ppi() - Select suitable PPI for the current system.
1119  *
1120  * If HYP mode is available, we know that the physical timer
1121  * has been configured to be accessible from PL1. Use it, so
1122  * that a guest can use the virtual timer instead.
1123  *
1124  * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
1125  * accesses to CNTP_*_EL1 registers are silently redirected to
1126  * their CNTHP_*_EL2 counterparts, and use a different PPI
1127  * number.
1128  *
1129  * If no interrupt provided for virtual timer, we'll have to
1130  * stick to the physical timer. It'd better be accessible...
1131  * For arm64 we never use the secure interrupt.
1132  *
1133  * Return: a suitable PPI type for the current system.
1134  */
1135 static enum arch_timer_ppi_nr __init arch_timer_select_ppi(void)
1136 {
1137         if (is_kernel_in_hyp_mode())
1138                 return ARCH_TIMER_HYP_PPI;
1139
1140         if (!is_hyp_mode_available() && arch_timer_ppi[ARCH_TIMER_VIRT_PPI])
1141                 return ARCH_TIMER_VIRT_PPI;
1142
1143         if (IS_ENABLED(CONFIG_ARM64))
1144                 return ARCH_TIMER_PHYS_NONSECURE_PPI;
1145
1146         return ARCH_TIMER_PHYS_SECURE_PPI;
1147 }
1148
1149 static int __init arch_timer_of_init(struct device_node *np)
1150 {
1151         int i, ret;
1152         u32 rate;
1153
1154         if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1155                 pr_warn("multiple nodes in dt, skipping\n");
1156                 return 0;
1157         }
1158
1159         arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1160         for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++)
1161                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
1162
1163         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1164
1165         rate = arch_timer_get_cntfrq();
1166         arch_timer_of_configure_rate(rate, np);
1167
1168         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
1169
1170         /* Check for globally applicable workarounds */
1171         arch_timer_check_ool_workaround(ate_match_dt, np);
1172
1173         /*
1174          * If we cannot rely on firmware initializing the timer registers then
1175          * we should use the physical timers instead.
1176          */
1177         if (IS_ENABLED(CONFIG_ARM) &&
1178             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
1179                 arch_timer_uses_ppi = ARCH_TIMER_PHYS_SECURE_PPI;
1180         else
1181                 arch_timer_uses_ppi = arch_timer_select_ppi();
1182
1183         if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1184                 pr_err("No interrupt available, giving up\n");
1185                 return -EINVAL;
1186         }
1187
1188         /* On some systems, the counter stops ticking when in suspend. */
1189         arch_counter_suspend_stop = of_property_read_bool(np,
1190                                                          "arm,no-tick-in-suspend");
1191
1192         ret = arch_timer_register();
1193         if (ret)
1194                 return ret;
1195
1196         if (arch_timer_needs_of_probing())
1197                 return 0;
1198
1199         return arch_timer_common_init();
1200 }
1201 TIMER_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
1202 TIMER_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
1203
1204 static u32 __init
1205 arch_timer_mem_frame_get_cntfrq(struct arch_timer_mem_frame *frame)
1206 {
1207         void __iomem *base;
1208         u32 rate;
1209
1210         base = ioremap(frame->cntbase, frame->size);
1211         if (!base) {
1212                 pr_err("Unable to map frame @ %pa\n", &frame->cntbase);
1213                 return 0;
1214         }
1215
1216         rate = readl_relaxed(base + CNTFRQ);
1217
1218         iounmap(base);
1219
1220         return rate;
1221 }
1222
1223 static struct arch_timer_mem_frame * __init
1224 arch_timer_mem_find_best_frame(struct arch_timer_mem *timer_mem)
1225 {
1226         struct arch_timer_mem_frame *frame, *best_frame = NULL;
1227         void __iomem *cntctlbase;
1228         u32 cnttidr;
1229         int i;
1230
1231         cntctlbase = ioremap(timer_mem->cntctlbase, timer_mem->size);
1232         if (!cntctlbase) {
1233                 pr_err("Can't map CNTCTLBase @ %pa\n",
1234                         &timer_mem->cntctlbase);
1235                 return NULL;
1236         }
1237
1238         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
1239
1240         /*
1241          * Try to find a virtual capable frame. Otherwise fall back to a
1242          * physical capable frame.
1243          */
1244         for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1245                 u32 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
1246                              CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
1247
1248                 frame = &timer_mem->frame[i];
1249                 if (!frame->valid)
1250                         continue;
1251
1252                 /* Try enabling everything, and see what sticks */
1253                 writel_relaxed(cntacr, cntctlbase + CNTACR(i));
1254                 cntacr = readl_relaxed(cntctlbase + CNTACR(i));
1255
1256                 if ((cnttidr & CNTTIDR_VIRT(i)) &&
1257                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
1258                         best_frame = frame;
1259                         arch_timer_mem_use_virtual = true;
1260                         break;
1261                 }
1262
1263                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
1264                         continue;
1265
1266                 best_frame = frame;
1267         }
1268
1269         iounmap(cntctlbase);
1270
1271         if (!best_frame)
1272                 pr_err("Unable to find a suitable frame in timer @ %pa\n",
1273                         &timer_mem->cntctlbase);
1274
1275         return best_frame;
1276 }
1277
1278 static int __init
1279 arch_timer_mem_frame_register(struct arch_timer_mem_frame *frame)
1280 {
1281         void __iomem *base;
1282         int ret, irq = 0;
1283
1284         if (arch_timer_mem_use_virtual)
1285                 irq = frame->virt_irq;
1286         else
1287                 irq = frame->phys_irq;
1288
1289         if (!irq) {
1290                 pr_err("Frame missing %s irq.\n",
1291                        arch_timer_mem_use_virtual ? "virt" : "phys");
1292                 return -EINVAL;
1293         }
1294
1295         if (!request_mem_region(frame->cntbase, frame->size,
1296                                 "arch_mem_timer"))
1297                 return -EBUSY;
1298
1299         base = ioremap(frame->cntbase, frame->size);
1300         if (!base) {
1301                 pr_err("Can't map frame's registers\n");
1302                 return -ENXIO;
1303         }
1304
1305         ret = arch_timer_mem_register(base, irq);
1306         if (ret) {
1307                 iounmap(base);
1308                 return ret;
1309         }
1310
1311         arch_counter_base = base;
1312         arch_timers_present |= ARCH_TIMER_TYPE_MEM;
1313
1314         return 0;
1315 }
1316
1317 static int __init arch_timer_mem_of_init(struct device_node *np)
1318 {
1319         struct arch_timer_mem *timer_mem;
1320         struct arch_timer_mem_frame *frame;
1321         struct device_node *frame_node;
1322         struct resource res;
1323         int ret = -EINVAL;
1324         u32 rate;
1325
1326         timer_mem = kzalloc(sizeof(*timer_mem), GFP_KERNEL);
1327         if (!timer_mem)
1328                 return -ENOMEM;
1329
1330         if (of_address_to_resource(np, 0, &res))
1331                 goto out;
1332         timer_mem->cntctlbase = res.start;
1333         timer_mem->size = resource_size(&res);
1334
1335         for_each_available_child_of_node(np, frame_node) {
1336                 u32 n;
1337                 struct arch_timer_mem_frame *frame;
1338
1339                 if (of_property_read_u32(frame_node, "frame-number", &n)) {
1340                         pr_err(FW_BUG "Missing frame-number.\n");
1341                         of_node_put(frame_node);
1342                         goto out;
1343                 }
1344                 if (n >= ARCH_TIMER_MEM_MAX_FRAMES) {
1345                         pr_err(FW_BUG "Wrong frame-number, only 0-%u are permitted.\n",
1346                                ARCH_TIMER_MEM_MAX_FRAMES - 1);
1347                         of_node_put(frame_node);
1348                         goto out;
1349                 }
1350                 frame = &timer_mem->frame[n];
1351
1352                 if (frame->valid) {
1353                         pr_err(FW_BUG "Duplicated frame-number.\n");
1354                         of_node_put(frame_node);
1355                         goto out;
1356                 }
1357
1358                 if (of_address_to_resource(frame_node, 0, &res)) {
1359                         of_node_put(frame_node);
1360                         goto out;
1361                 }
1362                 frame->cntbase = res.start;
1363                 frame->size = resource_size(&res);
1364
1365                 frame->virt_irq = irq_of_parse_and_map(frame_node,
1366                                                        ARCH_TIMER_VIRT_SPI);
1367                 frame->phys_irq = irq_of_parse_and_map(frame_node,
1368                                                        ARCH_TIMER_PHYS_SPI);
1369
1370                 frame->valid = true;
1371         }
1372
1373         frame = arch_timer_mem_find_best_frame(timer_mem);
1374         if (!frame) {
1375                 ret = -EINVAL;
1376                 goto out;
1377         }
1378
1379         rate = arch_timer_mem_frame_get_cntfrq(frame);
1380         arch_timer_of_configure_rate(rate, np);
1381
1382         ret = arch_timer_mem_frame_register(frame);
1383         if (!ret && !arch_timer_needs_of_probing())
1384                 ret = arch_timer_common_init();
1385 out:
1386         kfree(timer_mem);
1387         return ret;
1388 }
1389 TIMER_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
1390                        arch_timer_mem_of_init);
1391
1392 #ifdef CONFIG_ACPI_GTDT
1393 static int __init
1394 arch_timer_mem_verify_cntfrq(struct arch_timer_mem *timer_mem)
1395 {
1396         struct arch_timer_mem_frame *frame;
1397         u32 rate;
1398         int i;
1399
1400         for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1401                 frame = &timer_mem->frame[i];
1402
1403                 if (!frame->valid)
1404                         continue;
1405
1406                 rate = arch_timer_mem_frame_get_cntfrq(frame);
1407                 if (rate == arch_timer_rate)
1408                         continue;
1409
1410                 pr_err(FW_BUG "CNTFRQ mismatch: frame @ %pa: (0x%08lx), CPU: (0x%08lx)\n",
1411                         &frame->cntbase,
1412                         (unsigned long)rate, (unsigned long)arch_timer_rate);
1413
1414                 return -EINVAL;
1415         }
1416
1417         return 0;
1418 }
1419
1420 static int __init arch_timer_mem_acpi_init(int platform_timer_count)
1421 {
1422         struct arch_timer_mem *timers, *timer;
1423         struct arch_timer_mem_frame *frame;
1424         int timer_count, i, ret = 0;
1425
1426         timers = kcalloc(platform_timer_count, sizeof(*timers),
1427                             GFP_KERNEL);
1428         if (!timers)
1429                 return -ENOMEM;
1430
1431         ret = acpi_arch_timer_mem_init(timers, &timer_count);
1432         if (ret || !timer_count)
1433                 goto out;
1434
1435         for (i = 0; i < timer_count; i++) {
1436                 ret = arch_timer_mem_verify_cntfrq(&timers[i]);
1437                 if (ret) {
1438                         pr_err("Disabling MMIO timers due to CNTFRQ mismatch\n");
1439                         goto out;
1440                 }
1441         }
1442
1443         /*
1444          * While unlikely, it's theoretically possible that none of the frames
1445          * in a timer expose the combination of feature we want.
1446          */
1447         for (i = 0; i < timer_count; i++) {
1448                 timer = &timers[i];
1449
1450                 frame = arch_timer_mem_find_best_frame(timer);
1451                 if (frame)
1452                         break;
1453         }
1454
1455         if (frame)
1456                 ret = arch_timer_mem_frame_register(frame);
1457 out:
1458         kfree(timers);
1459         return ret;
1460 }
1461
1462 /* Initialize per-processor generic timer and memory-mapped timer(if present) */
1463 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1464 {
1465         int ret, platform_timer_count;
1466
1467         if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1468                 pr_warn("already initialized, skipping\n");
1469                 return -EINVAL;
1470         }
1471
1472         arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1473
1474         ret = acpi_gtdt_init(table, &platform_timer_count);
1475         if (ret) {
1476                 pr_err("Failed to init GTDT table.\n");
1477                 return ret;
1478         }
1479
1480         arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI] =
1481                 acpi_gtdt_map_ppi(ARCH_TIMER_PHYS_NONSECURE_PPI);
1482
1483         arch_timer_ppi[ARCH_TIMER_VIRT_PPI] =
1484                 acpi_gtdt_map_ppi(ARCH_TIMER_VIRT_PPI);
1485
1486         arch_timer_ppi[ARCH_TIMER_HYP_PPI] =
1487                 acpi_gtdt_map_ppi(ARCH_TIMER_HYP_PPI);
1488
1489         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1490
1491         /*
1492          * When probing via ACPI, we have no mechanism to override the sysreg
1493          * CNTFRQ value. This *must* be correct.
1494          */
1495         arch_timer_rate = arch_timer_get_cntfrq();
1496         if (!arch_timer_rate) {
1497                 pr_err(FW_BUG "frequency not available.\n");
1498                 return -EINVAL;
1499         }
1500
1501         arch_timer_uses_ppi = arch_timer_select_ppi();
1502         if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1503                 pr_err("No interrupt available, giving up\n");
1504                 return -EINVAL;
1505         }
1506
1507         /* Always-on capability */
1508         arch_timer_c3stop = acpi_gtdt_c3stop(arch_timer_uses_ppi);
1509
1510         /* Check for globally applicable workarounds */
1511         arch_timer_check_ool_workaround(ate_match_acpi_oem_info, table);
1512
1513         ret = arch_timer_register();
1514         if (ret)
1515                 return ret;
1516
1517         if (platform_timer_count &&
1518             arch_timer_mem_acpi_init(platform_timer_count))
1519                 pr_err("Failed to initialize memory-mapped timer.\n");
1520
1521         return arch_timer_common_init();
1522 }
1523 TIMER_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1524 #endif