drivers/clocksource/arm_arch_timer.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  linux/drivers/clocksource/arm_arch_timer.c
   4  *
   5  *  Copyright (C) 2011 ARM Ltd.
   6  *  All Rights Reserved
   7  */
   8
   9 #define pr_fmt(fmt)     "arch_timer: " fmt
  10
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/device.h>
  14 #include <linux/smp.h>
  15 #include <linux/cpu.h>
  16 #include <linux/cpu_pm.h>
  17 #include <linux/clockchips.h>
  18 #include <linux/clocksource.h>
  19 #include <linux/clocksource_ids.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/kstrtox.h>
  22 #include <linux/of_irq.h>
  23 #include <linux/of_address.h>
  24 #include <linux/io.h>
  25 #include <linux/slab.h>
  26 #include <linux/sched/clock.h>
  27 #include <linux/sched_clock.h>
  28 #include <linux/acpi.h>
  29 #include <linux/arm-smccc.h>
  30 #include <linux/ptp_kvm.h>
  31
  32 #include <asm/arch_timer.h>
  33 #include <asm/virt.h>
  34
  35 #include <clocksource/arm_arch_timer.h>
  36
  37 #define CNTTIDR         0x08
  38 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
  39
  40 #define CNTACR(n)       (0x40 + ((n) * 4))
  41 #define CNTACR_RPCT     BIT(0)
  42 #define CNTACR_RVCT     BIT(1)
  43 #define CNTACR_RFRQ     BIT(2)
  44 #define CNTACR_RVOFF    BIT(3)
  45 #define CNTACR_RWVT     BIT(4)
  46 #define CNTACR_RWPT     BIT(5)
  47
  48 #define CNTPCT_LO       0x00
  49 #define CNTVCT_LO       0x08
  50 #define CNTFRQ          0x10
  51 #define CNTP_CVAL_LO    0x20
  52 #define CNTP_CTL        0x2c
  53 #define CNTV_CVAL_LO    0x30
  54 #define CNTV_CTL        0x3c
  55
  56 /*
  57  * The minimum amount of time a generic counter is guaranteed to not roll over
  58  * (40 years)
  59  */
  60 #define MIN_ROLLOVER_SECS       (40ULL * 365 * 24 * 3600)
  61
  62 static unsigned arch_timers_present __initdata;
  63
  64 struct arch_timer {
  65         void __iomem *base;
  66         struct clock_event_device evt;
  67 };
  68
  69 static struct arch_timer *arch_timer_mem __ro_after_init;
  70
  71 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  72
  73 static u32 arch_timer_rate __ro_after_init;
  74 static int arch_timer_ppi[ARCH_TIMER_MAX_TIMER_PPI] __ro_after_init;
  75
  76 static const char *arch_timer_ppi_names[ARCH_TIMER_MAX_TIMER_PPI] = {
  77         [ARCH_TIMER_PHYS_SECURE_PPI]    = "sec-phys",
  78         [ARCH_TIMER_PHYS_NONSECURE_PPI] = "phys",
  79         [ARCH_TIMER_VIRT_PPI]           = "virt",
  80         [ARCH_TIMER_HYP_PPI]            = "hyp-phys",
  81         [ARCH_TIMER_HYP_VIRT_PPI]       = "hyp-virt",
  82 };
  83
  84 static struct clock_event_device __percpu *arch_timer_evt;
  85
  86 static enum arch_timer_ppi_nr arch_timer_uses_ppi __ro_after_init = ARCH_TIMER_VIRT_PPI;
  87 static bool arch_timer_c3stop __ro_after_init;
  88 static bool arch_timer_mem_use_virtual __ro_after_init;
  89 static bool arch_counter_suspend_stop __ro_after_init;
  90 #ifdef CONFIG_GENERIC_GETTIMEOFDAY
  91 static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_ARCHTIMER;
  92 #else
  93 static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_NONE;
  94 #endif /* CONFIG_GENERIC_GETTIMEOFDAY */
  95
  96 static cpumask_t evtstrm_available = CPU_MASK_NONE;
  97 static bool evtstrm_enable __ro_after_init = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
  98
  99 static int __init early_evtstrm_cfg(char *buf)
 100 {
 101         return kstrtobool(buf, &evtstrm_enable);
 102 }
 103 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
 104
 105 /*
 106  * Makes an educated guess at a valid counter width based on the Generic Timer
 107  * specification. Of note:
 108  *   1) the system counter is at least 56 bits wide
 109  *   2) a roll-over time of not less than 40 years
 110  *
 111  * See 'ARM DDI 0487G.a D11.1.2 ("The system counter")' for more details.
 112  */
 113 static int arch_counter_get_width(void)
 114 {
 115         u64 min_cycles = MIN_ROLLOVER_SECS * arch_timer_rate;
 116
 117         /* guarantee the returned width is within the valid range */
 118         return clamp_val(ilog2(min_cycles - 1) + 1, 56, 64);
 119 }
 120
 121 /*
 122  * Architected system timer support.
 123  */
 124
 125 static __always_inline
 126 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u64 val,
 127                           struct clock_event_device *clk)
 128 {
 129         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 130                 struct arch_timer *timer = to_arch_timer(clk);
 131                 switch (reg) {
 132                 case ARCH_TIMER_REG_CTRL:
 133                         writel_relaxed((u32)val, timer->base + CNTP_CTL);
 134                         break;
 135                 case ARCH_TIMER_REG_CVAL:
 136                         /*
 137                          * Not guaranteed to be atomic, so the timer
 138                          * must be disabled at this point.
 139                          */
 140                         writeq_relaxed(val, timer->base + CNTP_CVAL_LO);
 141                         break;
 142                 default:
 143                         BUILD_BUG();
 144                 }
 145         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 146                 struct arch_timer *timer = to_arch_timer(clk);
 147                 switch (reg) {
 148                 case ARCH_TIMER_REG_CTRL:
 149                         writel_relaxed((u32)val, timer->base + CNTV_CTL);
 150                         break;
 151                 case ARCH_TIMER_REG_CVAL:
 152                         /* Same restriction as above */
 153                         writeq_relaxed(val, timer->base + CNTV_CVAL_LO);
 154                         break;
 155                 default:
 156                         BUILD_BUG();
 157                 }
 158         } else {
 159                 arch_timer_reg_write_cp15(access, reg, val);
 160         }
 161 }
 162
 163 static __always_inline
 164 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 165                         struct clock_event_device *clk)
 166 {
 167         u32 val;
 168
 169         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 170                 struct arch_timer *timer = to_arch_timer(clk);
 171                 switch (reg) {
 172                 case ARCH_TIMER_REG_CTRL:
 173                         val = readl_relaxed(timer->base + CNTP_CTL);
 174                         break;
 175                 default:
 176                         BUILD_BUG();
 177                 }
 178         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 179                 struct arch_timer *timer = to_arch_timer(clk);
 180                 switch (reg) {
 181                 case ARCH_TIMER_REG_CTRL:
 182                         val = readl_relaxed(timer->base + CNTV_CTL);
 183                         break;
 184                 default:
 185                         BUILD_BUG();
 186                 }
 187         } else {
 188                 val = arch_timer_reg_read_cp15(access, reg);
 189         }
 190
 191         return val;
 192 }
 193
 194 static noinstr u64 raw_counter_get_cntpct_stable(void)
 195 {
 196         return __arch_counter_get_cntpct_stable();
 197 }
 198
 199 static notrace u64 arch_counter_get_cntpct_stable(void)
 200 {
 201         u64 val;
 202         preempt_disable_notrace();
 203         val = __arch_counter_get_cntpct_stable();
 204         preempt_enable_notrace();
 205         return val;
 206 }
 207
 208 static noinstr u64 arch_counter_get_cntpct(void)
 209 {
 210         return __arch_counter_get_cntpct();
 211 }
 212
 213 static noinstr u64 raw_counter_get_cntvct_stable(void)
 214 {
 215         return __arch_counter_get_cntvct_stable();
 216 }
 217
 218 static notrace u64 arch_counter_get_cntvct_stable(void)
 219 {
 220         u64 val;
 221         preempt_disable_notrace();
 222         val = __arch_counter_get_cntvct_stable();
 223         preempt_enable_notrace();
 224         return val;
 225 }
 226
 227 static noinstr u64 arch_counter_get_cntvct(void)
 228 {
 229         return __arch_counter_get_cntvct();
 230 }
 231
 232 /*
 233  * Default to cp15 based access because arm64 uses this function for
 234  * sched_clock() before DT is probed and the cp15 method is guaranteed
 235  * to exist on arm64. arm doesn't use this before DT is probed so even
 236  * if we don't have the cp15 accessors we won't have a problem.
 237  */
 238 u64 (*arch_timer_read_counter)(void) __ro_after_init = arch_counter_get_cntvct;
 239 EXPORT_SYMBOL_GPL(arch_timer_read_counter);
 240
 241 static u64 arch_counter_read(struct clocksource *cs)
 242 {
 243         return arch_timer_read_counter();
 244 }
 245
 246 static u64 arch_counter_read_cc(const struct cyclecounter *cc)
 247 {
 248         return arch_timer_read_counter();
 249 }
 250
 251 static struct clocksource clocksource_counter = {
 252         .name   = "arch_sys_counter",
 253         .id     = CSID_ARM_ARCH_COUNTER,
 254         .rating = 400,
 255         .read   = arch_counter_read,
 256         .flags  = CLOCK_SOURCE_IS_CONTINUOUS,
 257 };
 258
 259 static struct cyclecounter cyclecounter __ro_after_init = {
 260         .read   = arch_counter_read_cc,
 261 };
 262
 263 struct ate_acpi_oem_info {
 264         char oem_id[ACPI_OEM_ID_SIZE + 1];
 265         char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
 266         u32 oem_revision;
 267 };
 268
 269 #ifdef CONFIG_FSL_ERRATUM_A008585
 270 /*
 271  * The number of retries is an arbitrary value well beyond the highest number
 272  * of iterations the loop has been observed to take.
 273  */
 274 #define __fsl_a008585_read_reg(reg) ({                  \
 275         u64 _old, _new;                                 \
 276         int _retries = 200;                             \
 277                                                         \
 278         do {                                            \
 279                 _old = read_sysreg(reg);                \
 280                 _new = read_sysreg(reg);                \
 281                 _retries--;                             \
 282         } while (unlikely(_old != _new) && _retries);   \
 283                                                         \
 284         WARN_ON_ONCE(!_retries);                        \
 285         _new;                                           \
 286 })
 287
 288 static u64 notrace fsl_a008585_read_cntpct_el0(void)
 289 {
 290         return __fsl_a008585_read_reg(cntpct_el0);
 291 }
 292
 293 static u64 notrace fsl_a008585_read_cntvct_el0(void)
 294 {
 295         return __fsl_a008585_read_reg(cntvct_el0);
 296 }
 297 #endif
 298
 299 #ifdef CONFIG_HISILICON_ERRATUM_161010101
 300 /*
 301  * Verify whether the value of the second read is larger than the first by
 302  * less than 32 is the only way to confirm the value is correct, so clear the
 303  * lower 5 bits to check whether the difference is greater than 32 or not.
 304  * Theoretically the erratum should not occur more than twice in succession
 305  * when reading the system counter, but it is possible that some interrupts
 306  * may lead to more than twice read errors, triggering the warning, so setting
 307  * the number of retries far beyond the number of iterations the loop has been
 308  * observed to take.
 309  */
 310 #define __hisi_161010101_read_reg(reg) ({                               \
 311         u64 _old, _new;                                         \
 312         int _retries = 50;                                      \
 313                                                                 \
 314         do {                                                    \
 315                 _old = read_sysreg(reg);                        \
 316                 _new = read_sysreg(reg);                        \
 317                 _retries--;                                     \
 318         } while (unlikely((_new - _old) >> 5) && _retries);     \
 319                                                                 \
 320         WARN_ON_ONCE(!_retries);                                \
 321         _new;                                                   \
 322 })
 323
 324 static u64 notrace hisi_161010101_read_cntpct_el0(void)
 325 {
 326         return __hisi_161010101_read_reg(cntpct_el0);
 327 }
 328
 329 static u64 notrace hisi_161010101_read_cntvct_el0(void)
 330 {
 331         return __hisi_161010101_read_reg(cntvct_el0);
 332 }
 333
 334 static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {
 335         /*
 336          * Note that trailing spaces are required to properly match
 337          * the OEM table information.
 338          */
 339         {
 340                 .oem_id         = "HISI  ",
 341                 .oem_table_id   = "HIP05   ",
 342                 .oem_revision   = 0,
 343         },
 344         {
 345                 .oem_id         = "HISI  ",
 346                 .oem_table_id   = "HIP06   ",
 347                 .oem_revision   = 0,
 348         },
 349         {
 350                 .oem_id         = "HISI  ",
 351                 .oem_table_id   = "HIP07   ",
 352                 .oem_revision   = 0,
 353         },
 354         { /* Sentinel indicating the end of the OEM array */ },
 355 };
 356 #endif
 357
 358 #ifdef CONFIG_ARM64_ERRATUM_858921
 359 static u64 notrace arm64_858921_read_cntpct_el0(void)
 360 {
 361         u64 old, new;
 362
 363         old = read_sysreg(cntpct_el0);
 364         new = read_sysreg(cntpct_el0);
 365         return (((old ^ new) >> 32) & 1) ? old : new;
 366 }
 367
 368 static u64 notrace arm64_858921_read_cntvct_el0(void)
 369 {
 370         u64 old, new;
 371
 372         old = read_sysreg(cntvct_el0);
 373         new = read_sysreg(cntvct_el0);
 374         return (((old ^ new) >> 32) & 1) ? old : new;
 375 }
 376 #endif
 377
 378 #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
 379 /*
 380  * The low bits of the counter registers are indeterminate while bit 10 or
 381  * greater is rolling over. Since the counter value can jump both backward
 382  * (7ff -> 000 -> 800) and forward (7ff -> fff -> 800), ignore register values
 383  * with all ones or all zeros in the low bits. Bound the loop by the maximum
 384  * number of CPU cycles in 3 consecutive 24 MHz counter periods.
 385  */
 386 #define __sun50i_a64_read_reg(reg) ({                                   \
 387         u64 _val;                                                       \
 388         int _retries = 150;                                             \
 389                                                                         \
 390         do {                                                            \
 391                 _val = read_sysreg(reg);                                \
 392                 _retries--;                                             \
 393         } while (((_val + 1) & GENMASK(8, 0)) <= 1 && _retries);        \
 394                                                                         \
 395         WARN_ON_ONCE(!_retries);                                        \
 396         _val;                                                           \
 397 })
 398
 399 static u64 notrace sun50i_a64_read_cntpct_el0(void)
 400 {
 401         return __sun50i_a64_read_reg(cntpct_el0);
 402 }
 403
 404 static u64 notrace sun50i_a64_read_cntvct_el0(void)
 405 {
 406         return __sun50i_a64_read_reg(cntvct_el0);
 407 }
 408 #endif
 409
 410 #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
 411 DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround);
 412 EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround);
 413
 414 static atomic_t timer_unstable_counter_workaround_in_use = ATOMIC_INIT(0);
 415
 416 /*
 417  * Force the inlining of this function so that the register accesses
 418  * can be themselves correctly inlined.
 419  */
 420 static __always_inline
 421 void erratum_set_next_event_generic(const int access, unsigned long evt,
 422                                     struct clock_event_device *clk)
 423 {
 424         unsigned long ctrl;
 425         u64 cval;
 426
 427         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 428         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 429         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 430
 431         if (access == ARCH_TIMER_PHYS_ACCESS) {
 432                 cval = evt + arch_counter_get_cntpct_stable();
 433                 write_sysreg(cval, cntp_cval_el0);
 434         } else {
 435                 cval = evt + arch_counter_get_cntvct_stable();
 436                 write_sysreg(cval, cntv_cval_el0);
 437         }
 438
 439         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 440 }
 441
 442 static __maybe_unused int erratum_set_next_event_virt(unsigned long evt,
 443                                             struct clock_event_device *clk)
 444 {
 445         erratum_set_next_event_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 446         return 0;
 447 }
 448
 449 static __maybe_unused int erratum_set_next_event_phys(unsigned long evt,
 450                                             struct clock_event_device *clk)
 451 {
 452         erratum_set_next_event_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 453         return 0;
 454 }
 455
 456 static const struct arch_timer_erratum_workaround ool_workarounds[] = {
 457 #ifdef CONFIG_FSL_ERRATUM_A008585
 458         {
 459                 .match_type = ate_match_dt,
 460                 .id = "fsl,erratum-a008585",
 461                 .desc = "Freescale erratum a005858",
 462                 .read_cntpct_el0 = fsl_a008585_read_cntpct_el0,
 463                 .read_cntvct_el0 = fsl_a008585_read_cntvct_el0,
 464                 .set_next_event_phys = erratum_set_next_event_phys,
 465                 .set_next_event_virt = erratum_set_next_event_virt,
 466         },
 467 #endif
 468 #ifdef CONFIG_HISILICON_ERRATUM_161010101
 469         {
 470                 .match_type = ate_match_dt,
 471                 .id = "hisilicon,erratum-161010101",
 472                 .desc = "HiSilicon erratum 161010101",
 473                 .read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
 474                 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
 475                 .set_next_event_phys = erratum_set_next_event_phys,
 476                 .set_next_event_virt = erratum_set_next_event_virt,
 477         },
 478         {
 479                 .match_type = ate_match_acpi_oem_info,
 480                 .id = hisi_161010101_oem_info,
 481                 .desc = "HiSilicon erratum 161010101",
 482                 .read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
 483                 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
 484                 .set_next_event_phys = erratum_set_next_event_phys,
 485                 .set_next_event_virt = erratum_set_next_event_virt,
 486         },
 487 #endif
 488 #ifdef CONFIG_ARM64_ERRATUM_858921
 489         {
 490                 .match_type = ate_match_local_cap_id,
 491                 .id = (void *)ARM64_WORKAROUND_858921,
 492                 .desc = "ARM erratum 858921",
 493                 .read_cntpct_el0 = arm64_858921_read_cntpct_el0,
 494                 .read_cntvct_el0 = arm64_858921_read_cntvct_el0,
 495                 .set_next_event_phys = erratum_set_next_event_phys,
 496                 .set_next_event_virt = erratum_set_next_event_virt,
 497         },
 498 #endif
 499 #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
 500         {
 501                 .match_type = ate_match_dt,
 502                 .id = "allwinner,erratum-unknown1",
 503                 .desc = "Allwinner erratum UNKNOWN1",
 504                 .read_cntpct_el0 = sun50i_a64_read_cntpct_el0,
 505                 .read_cntvct_el0 = sun50i_a64_read_cntvct_el0,
 506                 .set_next_event_phys = erratum_set_next_event_phys,
 507                 .set_next_event_virt = erratum_set_next_event_virt,
 508         },
 509 #endif
 510 #ifdef CONFIG_ARM64_ERRATUM_1418040
 511         {
 512                 .match_type = ate_match_local_cap_id,
 513                 .id = (void *)ARM64_WORKAROUND_1418040,
 514                 .desc = "ARM erratum 1418040",
 515                 .disable_compat_vdso = true,
 516         },
 517 #endif
 518 };
 519
 520 typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,
 521                                const void *);
 522
 523 static
 524 bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa,
 525                                  const void *arg)
 526 {
 527         const struct device_node *np = arg;
 528
 529         return of_property_read_bool(np, wa->id);
 530 }
 531
 532 static
 533 bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa,
 534                                         const void *arg)
 535 {
 536         return this_cpu_has_cap((uintptr_t)wa->id);
 537 }
 538
 539
 540 static
 541 bool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa,
 542                                        const void *arg)
 543 {
 544         static const struct ate_acpi_oem_info empty_oem_info = {};
 545         const struct ate_acpi_oem_info *info = wa->id;
 546         const struct acpi_table_header *table = arg;
 547
 548         /* Iterate over the ACPI OEM info array, looking for a match */
 549         while (memcmp(info, &empty_oem_info, sizeof(*info))) {
 550                 if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) &&
 551                     !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
 552                     info->oem_revision == table->oem_revision)
 553                         return true;
 554
 555                 info++;
 556         }
 557
 558         return false;
 559 }
 560
 561 static const struct arch_timer_erratum_workaround *
 562 arch_timer_iterate_errata(enum arch_timer_erratum_match_type type,
 563                           ate_match_fn_t match_fn,
 564                           void *arg)
 565 {
 566         int i;
 567
 568         for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) {
 569                 if (ool_workarounds[i].match_type != type)
 570                         continue;
 571
 572                 if (match_fn(&ool_workarounds[i], arg))
 573                         return &ool_workarounds[i];
 574         }
 575
 576         return NULL;
 577 }
 578
 579 static
 580 void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa,
 581                                   bool local)
 582 {
 583         int i;
 584
 585         if (local) {
 586                 __this_cpu_write(timer_unstable_counter_workaround, wa);
 587         } else {
 588                 for_each_possible_cpu(i)
 589                         per_cpu(timer_unstable_counter_workaround, i) = wa;
 590         }
 591
 592         if (wa->read_cntvct_el0 || wa->read_cntpct_el0)
 593                 atomic_set(&timer_unstable_counter_workaround_in_use, 1);
 594
 595         /*
 596          * Don't use the vdso fastpath if errata require using the
 597          * out-of-line counter accessor. We may change our mind pretty
 598          * late in the game (with a per-CPU erratum, for example), so
 599          * change both the default value and the vdso itself.
 600          */
 601         if (wa->read_cntvct_el0) {
 602                 clocksource_counter.vdso_clock_mode = VDSO_CLOCKMODE_NONE;
 603                 vdso_default = VDSO_CLOCKMODE_NONE;
 604         } else if (wa->disable_compat_vdso && vdso_default != VDSO_CLOCKMODE_NONE) {
 605                 vdso_default = VDSO_CLOCKMODE_ARCHTIMER_NOCOMPAT;
 606                 clocksource_counter.vdso_clock_mode = vdso_default;
 607         }
 608 }
 609
 610 static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,
 611                                             void *arg)
 612 {
 613         const struct arch_timer_erratum_workaround *wa, *__wa;
 614         ate_match_fn_t match_fn = NULL;
 615         bool local = false;
 616
 617         switch (type) {
 618         case ate_match_dt:
 619                 match_fn = arch_timer_check_dt_erratum;
 620                 break;
 621         case ate_match_local_cap_id:
 622                 match_fn = arch_timer_check_local_cap_erratum;
 623                 local = true;
 624                 break;
 625         case ate_match_acpi_oem_info:
 626                 match_fn = arch_timer_check_acpi_oem_erratum;
 627                 break;
 628         default:
 629                 WARN_ON(1);
 630                 return;
 631         }
 632
 633         wa = arch_timer_iterate_errata(type, match_fn, arg);
 634         if (!wa)
 635                 return;
 636
 637         __wa = __this_cpu_read(timer_unstable_counter_workaround);
 638         if (__wa && wa != __wa)
 639                 pr_warn("Can't enable workaround for %s (clashes with %s\n)",
 640                         wa->desc, __wa->desc);
 641
 642         if (__wa)
 643                 return;
 644
 645         arch_timer_enable_workaround(wa, local);
 646         pr_info("Enabling %s workaround for %s\n",
 647                 local ? "local" : "global", wa->desc);
 648 }
 649
 650 static bool arch_timer_this_cpu_has_cntvct_wa(void)
 651 {
 652         return has_erratum_handler(read_cntvct_el0);
 653 }
 654
 655 static bool arch_timer_counter_has_wa(void)
 656 {
 657         return atomic_read(&timer_unstable_counter_workaround_in_use);
 658 }
 659 #else
 660 #define arch_timer_check_ool_workaround(t,a)            do { } while(0)
 661 #define arch_timer_this_cpu_has_cntvct_wa()             ({false;})
 662 #define arch_timer_counter_has_wa()                     ({false;})
 663 #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
 664
 665 static __always_inline irqreturn_t timer_handler(const int access,
 666                                         struct clock_event_device *evt)
 667 {
 668         unsigned long ctrl;
 669
 670         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 671         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 672                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 673                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 674                 evt->event_handler(evt);
 675                 return IRQ_HANDLED;
 676         }
 677
 678         return IRQ_NONE;
 679 }
 680
 681 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 682 {
 683         struct clock_event_device *evt = dev_id;
 684
 685         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 686 }
 687
 688 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 689 {
 690         struct clock_event_device *evt = dev_id;
 691
 692         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 693 }
 694
 695 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 696 {
 697         struct clock_event_device *evt = dev_id;
 698
 699         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 700 }
 701
 702 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 703 {
 704         struct clock_event_device *evt = dev_id;
 705
 706         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 707 }
 708
 709 static __always_inline int arch_timer_shutdown(const int access,
 710                                                struct clock_event_device *clk)
 711 {
 712         unsigned long ctrl;
 713
 714         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 715         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 716         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 717
 718         return 0;
 719 }
 720
 721 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
 722 {
 723         return arch_timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
 724 }
 725
 726 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
 727 {
 728         return arch_timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
 729 }
 730
 731 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
 732 {
 733         return arch_timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
 734 }
 735
 736 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
 737 {
 738         return arch_timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
 739 }
 740
 741 static __always_inline void set_next_event(const int access, unsigned long evt,
 742                                            struct clock_event_device *clk)
 743 {
 744         unsigned long ctrl;
 745         u64 cnt;
 746
 747         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 748         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 749         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 750
 751         if (access == ARCH_TIMER_PHYS_ACCESS)
 752                 cnt = __arch_counter_get_cntpct();
 753         else
 754                 cnt = __arch_counter_get_cntvct();
 755
 756         arch_timer_reg_write(access, ARCH_TIMER_REG_CVAL, evt + cnt, clk);
 757         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 758 }
 759
 760 static int arch_timer_set_next_event_virt(unsigned long evt,
 761                                           struct clock_event_device *clk)
 762 {
 763         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 764         return 0;
 765 }
 766
 767 static int arch_timer_set_next_event_phys(unsigned long evt,
 768                                           struct clock_event_device *clk)
 769 {
 770         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 771         return 0;
 772 }
 773
 774 static noinstr u64 arch_counter_get_cnt_mem(struct arch_timer *t, int offset_lo)
 775 {
 776         u32 cnt_lo, cnt_hi, tmp_hi;
 777
 778         do {
 779                 cnt_hi = __le32_to_cpu((__le32 __force)__raw_readl(t->base + offset_lo + 4));
 780                 cnt_lo = __le32_to_cpu((__le32 __force)__raw_readl(t->base + offset_lo));
 781                 tmp_hi = __le32_to_cpu((__le32 __force)__raw_readl(t->base + offset_lo + 4));
 782         } while (cnt_hi != tmp_hi);
 783
 784         return ((u64) cnt_hi << 32) | cnt_lo;
 785 }
 786
 787 static __always_inline void set_next_event_mem(const int access, unsigned long evt,
 788                                            struct clock_event_device *clk)
 789 {
 790         struct arch_timer *timer = to_arch_timer(clk);
 791         unsigned long ctrl;
 792         u64 cnt;
 793
 794         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 795
 796         /* Timer must be disabled before programming CVAL */
 797         if (ctrl & ARCH_TIMER_CTRL_ENABLE) {
 798                 ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 799                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 800         }
 801
 802         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 803         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 804
 805         if (access ==  ARCH_TIMER_MEM_VIRT_ACCESS)
 806                 cnt = arch_counter_get_cnt_mem(timer, CNTVCT_LO);
 807         else
 808                 cnt = arch_counter_get_cnt_mem(timer, CNTPCT_LO);
 809
 810         arch_timer_reg_write(access, ARCH_TIMER_REG_CVAL, evt + cnt, clk);
 811         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 812 }
 813
 814 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 815                                               struct clock_event_device *clk)
 816 {
 817         set_next_event_mem(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 818         return 0;
 819 }
 820
 821 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 822                                               struct clock_event_device *clk)
 823 {
 824         set_next_event_mem(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 825         return 0;
 826 }
 827
 828 static u64 __arch_timer_check_delta(void)
 829 {
 830 #ifdef CONFIG_ARM64
 831         const struct midr_range broken_cval_midrs[] = {
 832                 /*
 833                  * XGene-1 implements CVAL in terms of TVAL, meaning
 834                  * that the maximum timer range is 32bit. Shame on them.
 835                  *
 836                  * Note that TVAL is signed, thus has only 31 of its
 837                  * 32 bits to express magnitude.
 838                  */
 839                 MIDR_ALL_VERSIONS(MIDR_CPU_MODEL(ARM_CPU_IMP_APM,
 840                                                  APM_CPU_PART_POTENZA)),
 841                 {},
 842         };
 843
 844         if (is_midr_in_range_list(read_cpuid_id(), broken_cval_midrs)) {
 845                 pr_warn_once("Broken CNTx_CVAL_EL1, using 31 bit TVAL instead.\n");
 846                 return CLOCKSOURCE_MASK(31);
 847         }
 848 #endif
 849         return CLOCKSOURCE_MASK(arch_counter_get_width());
 850 }
 851
 852 static void __arch_timer_setup(unsigned type,
 853                                struct clock_event_device *clk)
 854 {
 855         u64 max_delta;
 856
 857         clk->features = CLOCK_EVT_FEAT_ONESHOT;
 858
 859         if (type == ARCH_TIMER_TYPE_CP15) {
 860                 typeof(clk->set_next_event) sne;
 861
 862                 arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);
 863
 864                 if (arch_timer_c3stop)
 865                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
 866                 clk->name = "arch_sys_timer";
 867                 clk->rating = 450;
 868                 clk->cpumask = cpumask_of(smp_processor_id());
 869                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
 870                 switch (arch_timer_uses_ppi) {
 871                 case ARCH_TIMER_VIRT_PPI:
 872                         clk->set_state_shutdown = arch_timer_shutdown_virt;
 873                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
 874                         sne = erratum_handler(set_next_event_virt);
 875                         break;
 876                 case ARCH_TIMER_PHYS_SECURE_PPI:
 877                 case ARCH_TIMER_PHYS_NONSECURE_PPI:
 878                 case ARCH_TIMER_HYP_PPI:
 879                         clk->set_state_shutdown = arch_timer_shutdown_phys;
 880                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
 881                         sne = erratum_handler(set_next_event_phys);
 882                         break;
 883                 default:
 884                         BUG();
 885                 }
 886
 887                 clk->set_next_event = sne;
 888                 max_delta = __arch_timer_check_delta();
 889         } else {
 890                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 891                 clk->name = "arch_mem_timer";
 892                 clk->rating = 400;
 893                 clk->cpumask = cpu_possible_mask;
 894                 if (arch_timer_mem_use_virtual) {
 895                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
 896                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
 897                         clk->set_next_event =
 898                                 arch_timer_set_next_event_virt_mem;
 899                 } else {
 900                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
 901                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
 902                         clk->set_next_event =
 903                                 arch_timer_set_next_event_phys_mem;
 904                 }
 905
 906                 max_delta = CLOCKSOURCE_MASK(56);
 907         }
 908
 909         clk->set_state_shutdown(clk);
 910
 911         clockevents_config_and_register(clk, arch_timer_rate, 0xf, max_delta);
 912 }
 913
 914 static void arch_timer_evtstrm_enable(unsigned int divider)
 915 {
 916         u32 cntkctl = arch_timer_get_cntkctl();
 917
 918 #ifdef CONFIG_ARM64
 919         /* ECV is likely to require a large divider. Use the EVNTIS flag. */
 920         if (cpus_have_const_cap(ARM64_HAS_ECV) && divider > 15) {
 921                 cntkctl |= ARCH_TIMER_EVT_INTERVAL_SCALE;
 922                 divider -= 8;
 923         }
 924 #endif
 925
 926         divider = min(divider, 15U);
 927         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 928         /* Set the divider and enable virtual event stream */
 929         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 930                         | ARCH_TIMER_VIRT_EVT_EN;
 931         arch_timer_set_cntkctl(cntkctl);
 932         arch_timer_set_evtstrm_feature();
 933         cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
 934 }
 935
 936 static void arch_timer_configure_evtstream(void)
 937 {
 938         int evt_stream_div, lsb;
 939
 940         /*
 941          * As the event stream can at most be generated at half the frequency
 942          * of the counter, use half the frequency when computing the divider.
 943          */
 944         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ / 2;
 945
 946         /*
 947          * Find the closest power of two to the divisor. If the adjacent bit
 948          * of lsb (last set bit, starts from 0) is set, then we use (lsb + 1).
 949          */
 950         lsb = fls(evt_stream_div) - 1;
 951         if (lsb > 0 && (evt_stream_div & BIT(lsb - 1)))
 952                 lsb++;
 953
 954         /* enable event stream */
 955         arch_timer_evtstrm_enable(max(0, lsb));
 956 }
 957
 958 static void arch_counter_set_user_access(void)
 959 {
 960         u32 cntkctl = arch_timer_get_cntkctl();
 961
 962         /* Disable user access to the timers and both counters */
 963         /* Also disable virtual event stream */
 964         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 965                         | ARCH_TIMER_USR_VT_ACCESS_EN
 966                         | ARCH_TIMER_USR_VCT_ACCESS_EN
 967                         | ARCH_TIMER_VIRT_EVT_EN
 968                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
 969
 970         /*
 971          * Enable user access to the virtual counter if it doesn't
 972          * need to be workaround. The vdso may have been already
 973          * disabled though.
 974          */
 975         if (arch_timer_this_cpu_has_cntvct_wa())
 976                 pr_info("CPU%d: Trapping CNTVCT access\n", smp_processor_id());
 977         else
 978                 cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 979
 980         arch_timer_set_cntkctl(cntkctl);
 981 }
 982
 983 static bool arch_timer_has_nonsecure_ppi(void)
 984 {
 985         return (arch_timer_uses_ppi == ARCH_TIMER_PHYS_SECURE_PPI &&
 986                 arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
 987 }
 988
 989 static u32 check_ppi_trigger(int irq)
 990 {
 991         u32 flags = irq_get_trigger_type(irq);
 992
 993         if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
 994                 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
 995                 pr_warn("WARNING: Please fix your firmware\n");
 996                 flags = IRQF_TRIGGER_LOW;
 997         }
 998
 999         return flags;
1000 }
1001
1002 static int arch_timer_starting_cpu(unsigned int cpu)
1003 {
1004         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
1005         u32 flags;
1006
1007         __arch_timer_setup(ARCH_TIMER_TYPE_CP15, clk);
1008
1009         flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
1010         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
1011
1012         if (arch_timer_has_nonsecure_ppi()) {
1013                 flags = check_ppi_trigger(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
1014                 enable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
1015                                   flags);
1016         }
1017
1018         arch_counter_set_user_access();
1019         if (evtstrm_enable)
1020                 arch_timer_configure_evtstream();
1021
1022         return 0;
1023 }
1024
1025 static int validate_timer_rate(void)
1026 {
1027         if (!arch_timer_rate)
1028                 return -EINVAL;
1029
1030         /* Arch timer frequency < 1MHz can cause trouble */
1031         WARN_ON(arch_timer_rate < 1000000);
1032
1033         return 0;
1034 }
1035
1036 /*
1037  * For historical reasons, when probing with DT we use whichever (non-zero)
1038  * rate was probed first, and don't verify that others match. If the first node
1039  * probed has a clock-frequency property, this overrides the HW register.
1040  */
1041 static void __init arch_timer_of_configure_rate(u32 rate, struct device_node *np)
1042 {
1043         /* Who has more than one independent system counter? */
1044         if (arch_timer_rate)
1045                 return;
1046
1047         if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate))
1048                 arch_timer_rate = rate;
1049
1050         /* Check the timer frequency. */
1051         if (validate_timer_rate())
1052                 pr_warn("frequency not available\n");
1053 }
1054
1055 static void __init arch_timer_banner(unsigned type)
1056 {
1057         pr_info("%s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
1058                 type & ARCH_TIMER_TYPE_CP15 ? "cp15" : "",
1059                 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ?
1060                         " and " : "",
1061                 type & ARCH_TIMER_TYPE_MEM ? "mmio" : "",
1062                 (unsigned long)arch_timer_rate / 1000000,
1063                 (unsigned long)(arch_timer_rate / 10000) % 100,
1064                 type & ARCH_TIMER_TYPE_CP15 ?
1065                         (arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) ? "virt" : "phys" :
1066                         "",
1067                 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? "/" : "",
1068                 type & ARCH_TIMER_TYPE_MEM ?
1069                         arch_timer_mem_use_virtual ? "virt" : "phys" :
1070                         "");
1071 }
1072
1073 u32 arch_timer_get_rate(void)
1074 {
1075         return arch_timer_rate;
1076 }
1077
1078 bool arch_timer_evtstrm_available(void)
1079 {
1080         /*
1081          * We might get called from a preemptible context. This is fine
1082          * because availability of the event stream should be always the same
1083          * for a preemptible context and context where we might resume a task.
1084          */
1085         return cpumask_test_cpu(raw_smp_processor_id(), &evtstrm_available);
1086 }
1087
1088 static noinstr u64 arch_counter_get_cntvct_mem(void)
1089 {
1090         return arch_counter_get_cnt_mem(arch_timer_mem, CNTVCT_LO);
1091 }
1092
1093 static struct arch_timer_kvm_info arch_timer_kvm_info;
1094
1095 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
1096 {
1097         return &arch_timer_kvm_info;
1098 }
1099
1100 static void __init arch_counter_register(unsigned type)
1101 {
1102         u64 (*scr)(void);
1103         u64 start_count;
1104         int width;
1105
1106         /* Register the CP15 based counter if we have one */
1107         if (type & ARCH_TIMER_TYPE_CP15) {
1108                 u64 (*rd)(void);
1109
1110                 if ((IS_ENABLED(CONFIG_ARM64) && !is_hyp_mode_available()) ||
1111                     arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) {
1112                         if (arch_timer_counter_has_wa()) {
1113                                 rd = arch_counter_get_cntvct_stable;
1114                                 scr = raw_counter_get_cntvct_stable;
1115                         } else {
1116                                 rd = arch_counter_get_cntvct;
1117                                 scr = arch_counter_get_cntvct;
1118                         }
1119                 } else {
1120                         if (arch_timer_counter_has_wa()) {
1121                                 rd = arch_counter_get_cntpct_stable;
1122                                 scr = raw_counter_get_cntpct_stable;
1123                         } else {
1124                                 rd = arch_counter_get_cntpct;
1125                                 scr = arch_counter_get_cntpct;
1126                         }
1127                 }
1128
1129                 arch_timer_read_counter = rd;
1130                 clocksource_counter.vdso_clock_mode = vdso_default;
1131         } else {
1132                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
1133                 scr = arch_counter_get_cntvct_mem;
1134         }
1135
1136         width = arch_counter_get_width();
1137         clocksource_counter.mask = CLOCKSOURCE_MASK(width);
1138         cyclecounter.mask = CLOCKSOURCE_MASK(width);
1139
1140         if (!arch_counter_suspend_stop)
1141                 clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
1142         start_count = arch_timer_read_counter();
1143         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
1144         cyclecounter.mult = clocksource_counter.mult;
1145         cyclecounter.shift = clocksource_counter.shift;
1146         timecounter_init(&arch_timer_kvm_info.timecounter,
1147                          &cyclecounter, start_count);
1148
1149         sched_clock_register(scr, width, arch_timer_rate);
1150 }
1151
1152 static void arch_timer_stop(struct clock_event_device *clk)
1153 {
1154         pr_debug("disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id());
1155
1156         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
1157         if (arch_timer_has_nonsecure_ppi())
1158                 disable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
1159
1160         clk->set_state_shutdown(clk);
1161 }
1162
1163 static int arch_timer_dying_cpu(unsigned int cpu)
1164 {
1165         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
1166
1167         cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1168
1169         arch_timer_stop(clk);
1170         return 0;
1171 }
1172
1173 #ifdef CONFIG_CPU_PM
1174 static DEFINE_PER_CPU(unsigned long, saved_cntkctl);
1175 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
1176                                     unsigned long action, void *hcpu)
1177 {
1178         if (action == CPU_PM_ENTER) {
1179                 __this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl());
1180
1181                 cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1182         } else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) {
1183                 arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
1184
1185                 if (arch_timer_have_evtstrm_feature())
1186                         cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
1187         }
1188         return NOTIFY_OK;
1189 }
1190
1191 static struct notifier_block arch_timer_cpu_pm_notifier = {
1192         .notifier_call = arch_timer_cpu_pm_notify,
1193 };
1194
1195 static int __init arch_timer_cpu_pm_init(void)
1196 {
1197         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
1198 }
1199
1200 static void __init arch_timer_cpu_pm_deinit(void)
1201 {
1202         WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
1203 }
1204
1205 #else
1206 static int __init arch_timer_cpu_pm_init(void)
1207 {
1208         return 0;
1209 }
1210
1211 static void __init arch_timer_cpu_pm_deinit(void)
1212 {
1213 }
1214 #endif
1215
1216 static int __init arch_timer_register(void)
1217 {
1218         int err;
1219         int ppi;
1220
1221         arch_timer_evt = alloc_percpu(struct clock_event_device);
1222         if (!arch_timer_evt) {
1223                 err = -ENOMEM;
1224                 goto out;
1225         }
1226
1227         ppi = arch_timer_ppi[arch_timer_uses_ppi];
1228         switch (arch_timer_uses_ppi) {
1229         case ARCH_TIMER_VIRT_PPI:
1230                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
1231                                          "arch_timer", arch_timer_evt);
1232                 break;
1233         case ARCH_TIMER_PHYS_SECURE_PPI:
1234         case ARCH_TIMER_PHYS_NONSECURE_PPI:
1235                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
1236                                          "arch_timer", arch_timer_evt);
1237                 if (!err && arch_timer_has_nonsecure_ppi()) {
1238                         ppi = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1239                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
1240                                                  "arch_timer", arch_timer_evt);
1241                         if (err)
1242                                 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_SECURE_PPI],
1243                                                 arch_timer_evt);
1244                 }
1245                 break;
1246         case ARCH_TIMER_HYP_PPI:
1247                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
1248                                          "arch_timer", arch_timer_evt);
1249                 break;
1250         default:
1251                 BUG();
1252         }
1253
1254         if (err) {
1255                 pr_err("can't register interrupt %d (%d)\n", ppi, err);
1256                 goto out_free;
1257         }
1258
1259         err = arch_timer_cpu_pm_init();
1260         if (err)
1261                 goto out_unreg_notify;
1262
1263         /* Register and immediately configure the timer on the boot CPU */
1264         err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
1265                                 "clockevents/arm/arch_timer:starting",
1266                                 arch_timer_starting_cpu, arch_timer_dying_cpu);
1267         if (err)
1268                 goto out_unreg_cpupm;
1269         return 0;
1270
1271 out_unreg_cpupm:
1272         arch_timer_cpu_pm_deinit();
1273
1274 out_unreg_notify:
1275         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
1276         if (arch_timer_has_nonsecure_ppi())
1277                 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
1278                                 arch_timer_evt);
1279
1280 out_free:
1281         free_percpu(arch_timer_evt);
1282 out:
1283         return err;
1284 }
1285
1286 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
1287 {
1288         int ret;
1289         irq_handler_t func;
1290
1291         arch_timer_mem = kzalloc(sizeof(*arch_timer_mem), GFP_KERNEL);
1292         if (!arch_timer_mem)
1293                 return -ENOMEM;
1294
1295         arch_timer_mem->base = base;
1296         arch_timer_mem->evt.irq = irq;
1297         __arch_timer_setup(ARCH_TIMER_TYPE_MEM, &arch_timer_mem->evt);
1298
1299         if (arch_timer_mem_use_virtual)
1300                 func = arch_timer_handler_virt_mem;
1301         else
1302                 func = arch_timer_handler_phys_mem;
1303
1304         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &arch_timer_mem->evt);
1305         if (ret) {
1306                 pr_err("Failed to request mem timer irq\n");
1307                 kfree(arch_timer_mem);
1308                 arch_timer_mem = NULL;
1309         }
1310
1311         return ret;
1312 }
1313
1314 static const struct of_device_id arch_timer_of_match[] __initconst = {
1315         { .compatible   = "arm,armv7-timer",    },
1316         { .compatible   = "arm,armv8-timer",    },
1317         {},
1318 };
1319
1320 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
1321         { .compatible   = "arm,armv7-timer-mem", },
1322         {},
1323 };
1324
1325 static bool __init arch_timer_needs_of_probing(void)
1326 {
1327         struct device_node *dn;
1328         bool needs_probing = false;
1329         unsigned int mask = ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM;
1330
1331         /* We have two timers, and both device-tree nodes are probed. */
1332         if ((arch_timers_present & mask) == mask)
1333                 return false;
1334
1335         /*
1336          * Only one type of timer is probed,
1337          * check if we have another type of timer node in device-tree.
1338          */
1339         if (arch_timers_present & ARCH_TIMER_TYPE_CP15)
1340                 dn = of_find_matching_node(NULL, arch_timer_mem_of_match);
1341         else
1342                 dn = of_find_matching_node(NULL, arch_timer_of_match);
1343
1344         if (dn && of_device_is_available(dn))
1345                 needs_probing = true;
1346
1347         of_node_put(dn);
1348
1349         return needs_probing;
1350 }
1351
1352 static int __init arch_timer_common_init(void)
1353 {
1354         arch_timer_banner(arch_timers_present);
1355         arch_counter_register(arch_timers_present);
1356         return arch_timer_arch_init();
1357 }
1358
1359 /**
1360  * arch_timer_select_ppi() - Select suitable PPI for the current system.
1361  *
1362  * If HYP mode is available, we know that the physical timer
1363  * has been configured to be accessible from PL1. Use it, so
1364  * that a guest can use the virtual timer instead.
1365  *
1366  * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
1367  * accesses to CNTP_*_EL1 registers are silently redirected to
1368  * their CNTHP_*_EL2 counterparts, and use a different PPI
1369  * number.
1370  *
1371  * If no interrupt provided for virtual timer, we'll have to
1372  * stick to the physical timer. It'd better be accessible...
1373  * For arm64 we never use the secure interrupt.
1374  *
1375  * Return: a suitable PPI type for the current system.
1376  */
1377 static enum arch_timer_ppi_nr __init arch_timer_select_ppi(void)
1378 {
1379         if (is_kernel_in_hyp_mode())
1380                 return ARCH_TIMER_HYP_PPI;
1381
1382         if (!is_hyp_mode_available() && arch_timer_ppi[ARCH_TIMER_VIRT_PPI])
1383                 return ARCH_TIMER_VIRT_PPI;
1384
1385         if (IS_ENABLED(CONFIG_ARM64))
1386                 return ARCH_TIMER_PHYS_NONSECURE_PPI;
1387
1388         return ARCH_TIMER_PHYS_SECURE_PPI;
1389 }
1390
1391 static void __init arch_timer_populate_kvm_info(void)
1392 {
1393         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1394         if (is_kernel_in_hyp_mode())
1395                 arch_timer_kvm_info.physical_irq = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1396 }
1397
1398 static int __init arch_timer_of_init(struct device_node *np)
1399 {
1400         int i, irq, ret;
1401         u32 rate;
1402         bool has_names;
1403
1404         if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1405                 pr_warn("multiple nodes in dt, skipping\n");
1406                 return 0;
1407         }
1408
1409         arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1410
1411         has_names = of_property_read_bool(np, "interrupt-names");
1412
1413         for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++) {
1414                 if (has_names)
1415                         irq = of_irq_get_byname(np, arch_timer_ppi_names[i]);
1416                 else
1417                         irq = of_irq_get(np, i);
1418                 if (irq > 0)
1419                         arch_timer_ppi[i] = irq;
1420         }
1421
1422         arch_timer_populate_kvm_info();
1423
1424         rate = arch_timer_get_cntfrq();
1425         arch_timer_of_configure_rate(rate, np);
1426
1427         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
1428
1429         /* Check for globally applicable workarounds */
1430         arch_timer_check_ool_workaround(ate_match_dt, np);
1431
1432         /*
1433          * If we cannot rely on firmware initializing the timer registers then
1434          * we should use the physical timers instead.
1435          */
1436         if (IS_ENABLED(CONFIG_ARM) &&
1437             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
1438                 arch_timer_uses_ppi = ARCH_TIMER_PHYS_SECURE_PPI;
1439         else
1440                 arch_timer_uses_ppi = arch_timer_select_ppi();
1441
1442         if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1443                 pr_err("No interrupt available, giving up\n");
1444                 return -EINVAL;
1445         }
1446
1447         /* On some systems, the counter stops ticking when in suspend. */
1448         arch_counter_suspend_stop = of_property_read_bool(np,
1449                                                          "arm,no-tick-in-suspend");
1450
1451         ret = arch_timer_register();
1452         if (ret)
1453                 return ret;
1454
1455         if (arch_timer_needs_of_probing())
1456                 return 0;
1457
1458         return arch_timer_common_init();
1459 }
1460 TIMER_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
1461 TIMER_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
1462
1463 static u32 __init
1464 arch_timer_mem_frame_get_cntfrq(struct arch_timer_mem_frame *frame)
1465 {
1466         void __iomem *base;
1467         u32 rate;
1468
1469         base = ioremap(frame->cntbase, frame->size);
1470         if (!base) {
1471                 pr_err("Unable to map frame @ %pa\n", &frame->cntbase);
1472                 return 0;
1473         }
1474
1475         rate = readl_relaxed(base + CNTFRQ);
1476
1477         iounmap(base);
1478
1479         return rate;
1480 }
1481
1482 static struct arch_timer_mem_frame * __init
1483 arch_timer_mem_find_best_frame(struct arch_timer_mem *timer_mem)
1484 {
1485         struct arch_timer_mem_frame *frame, *best_frame = NULL;
1486         void __iomem *cntctlbase;
1487         u32 cnttidr;
1488         int i;
1489
1490         cntctlbase = ioremap(timer_mem->cntctlbase, timer_mem->size);
1491         if (!cntctlbase) {
1492                 pr_err("Can't map CNTCTLBase @ %pa\n",
1493                         &timer_mem->cntctlbase);
1494                 return NULL;
1495         }
1496
1497         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
1498
1499         /*
1500          * Try to find a virtual capable frame. Otherwise fall back to a
1501          * physical capable frame.
1502          */
1503         for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1504                 u32 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
1505                              CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
1506
1507                 frame = &timer_mem->frame[i];
1508                 if (!frame->valid)
1509                         continue;
1510
1511                 /* Try enabling everything, and see what sticks */
1512                 writel_relaxed(cntacr, cntctlbase + CNTACR(i));
1513                 cntacr = readl_relaxed(cntctlbase + CNTACR(i));
1514
1515                 if ((cnttidr & CNTTIDR_VIRT(i)) &&
1516                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
1517                         best_frame = frame;
1518                         arch_timer_mem_use_virtual = true;
1519                         break;
1520                 }
1521
1522                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
1523                         continue;
1524
1525                 best_frame = frame;
1526         }
1527
1528         iounmap(cntctlbase);
1529
1530         return best_frame;
1531 }
1532
1533 static int __init
1534 arch_timer_mem_frame_register(struct arch_timer_mem_frame *frame)
1535 {
1536         void __iomem *base;
1537         int ret, irq = 0;
1538
1539         if (arch_timer_mem_use_virtual)
1540                 irq = frame->virt_irq;
1541         else
1542                 irq = frame->phys_irq;
1543
1544         if (!irq) {
1545                 pr_err("Frame missing %s irq.\n",
1546                        arch_timer_mem_use_virtual ? "virt" : "phys");
1547                 return -EINVAL;
1548         }
1549
1550         if (!request_mem_region(frame->cntbase, frame->size,
1551                                 "arch_mem_timer"))
1552                 return -EBUSY;
1553
1554         base = ioremap(frame->cntbase, frame->size);
1555         if (!base) {
1556                 pr_err("Can't map frame's registers\n");
1557                 return -ENXIO;
1558         }
1559
1560         ret = arch_timer_mem_register(base, irq);
1561         if (ret) {
1562                 iounmap(base);
1563                 return ret;
1564         }
1565
1566         arch_timers_present |= ARCH_TIMER_TYPE_MEM;
1567
1568         return 0;
1569 }
1570
1571 static int __init arch_timer_mem_of_init(struct device_node *np)
1572 {
1573         struct arch_timer_mem *timer_mem;
1574         struct arch_timer_mem_frame *frame;
1575         struct device_node *frame_node;
1576         struct resource res;
1577         int ret = -EINVAL;
1578         u32 rate;
1579
1580         timer_mem = kzalloc(sizeof(*timer_mem), GFP_KERNEL);
1581         if (!timer_mem)
1582                 return -ENOMEM;
1583
1584         if (of_address_to_resource(np, 0, &res))
1585                 goto out;
1586         timer_mem->cntctlbase = res.start;
1587         timer_mem->size = resource_size(&res);
1588
1589         for_each_available_child_of_node(np, frame_node) {
1590                 u32 n;
1591                 struct arch_timer_mem_frame *frame;
1592
1593                 if (of_property_read_u32(frame_node, "frame-number", &n)) {
1594                         pr_err(FW_BUG "Missing frame-number.\n");
1595                         of_node_put(frame_node);
1596                         goto out;
1597                 }
1598                 if (n >= ARCH_TIMER_MEM_MAX_FRAMES) {
1599                         pr_err(FW_BUG "Wrong frame-number, only 0-%u are permitted.\n",
1600                                ARCH_TIMER_MEM_MAX_FRAMES - 1);
1601                         of_node_put(frame_node);
1602                         goto out;
1603                 }
1604                 frame = &timer_mem->frame[n];
1605
1606                 if (frame->valid) {
1607                         pr_err(FW_BUG "Duplicated frame-number.\n");
1608                         of_node_put(frame_node);
1609                         goto out;
1610                 }
1611
1612                 if (of_address_to_resource(frame_node, 0, &res)) {
1613                         of_node_put(frame_node);
1614                         goto out;
1615                 }
1616                 frame->cntbase = res.start;
1617                 frame->size = resource_size(&res);
1618
1619                 frame->virt_irq = irq_of_parse_and_map(frame_node,
1620                                                        ARCH_TIMER_VIRT_SPI);
1621                 frame->phys_irq = irq_of_parse_and_map(frame_node,
1622                                                        ARCH_TIMER_PHYS_SPI);
1623
1624                 frame->valid = true;
1625         }
1626
1627         frame = arch_timer_mem_find_best_frame(timer_mem);
1628         if (!frame) {
1629                 pr_err("Unable to find a suitable frame in timer @ %pa\n",
1630                         &timer_mem->cntctlbase);
1631                 ret = -EINVAL;
1632                 goto out;
1633         }
1634
1635         rate = arch_timer_mem_frame_get_cntfrq(frame);
1636         arch_timer_of_configure_rate(rate, np);
1637
1638         ret = arch_timer_mem_frame_register(frame);
1639         if (!ret && !arch_timer_needs_of_probing())
1640                 ret = arch_timer_common_init();
1641 out:
1642         kfree(timer_mem);
1643         return ret;
1644 }
1645 TIMER_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
1646                        arch_timer_mem_of_init);
1647
1648 #ifdef CONFIG_ACPI_GTDT
1649 static int __init
1650 arch_timer_mem_verify_cntfrq(struct arch_timer_mem *timer_mem)
1651 {
1652         struct arch_timer_mem_frame *frame;
1653         u32 rate;
1654         int i;
1655
1656         for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1657                 frame = &timer_mem->frame[i];
1658
1659                 if (!frame->valid)
1660                         continue;
1661
1662                 rate = arch_timer_mem_frame_get_cntfrq(frame);
1663                 if (rate == arch_timer_rate)
1664                         continue;
1665
1666                 pr_err(FW_BUG "CNTFRQ mismatch: frame @ %pa: (0x%08lx), CPU: (0x%08lx)\n",
1667                         &frame->cntbase,
1668                         (unsigned long)rate, (unsigned long)arch_timer_rate);
1669
1670                 return -EINVAL;
1671         }
1672
1673         return 0;
1674 }
1675
1676 static int __init arch_timer_mem_acpi_init(int platform_timer_count)
1677 {
1678         struct arch_timer_mem *timers, *timer;
1679         struct arch_timer_mem_frame *frame, *best_frame = NULL;
1680         int timer_count, i, ret = 0;
1681
1682         timers = kcalloc(platform_timer_count, sizeof(*timers),
1683                             GFP_KERNEL);
1684         if (!timers)
1685                 return -ENOMEM;
1686
1687         ret = acpi_arch_timer_mem_init(timers, &timer_count);
1688         if (ret || !timer_count)
1689                 goto out;
1690
1691         /*
1692          * While unlikely, it's theoretically possible that none of the frames
1693          * in a timer expose the combination of feature we want.
1694          */
1695         for (i = 0; i < timer_count; i++) {
1696                 timer = &timers[i];
1697
1698                 frame = arch_timer_mem_find_best_frame(timer);
1699                 if (!best_frame)
1700                         best_frame = frame;
1701
1702                 ret = arch_timer_mem_verify_cntfrq(timer);
1703                 if (ret) {
1704                         pr_err("Disabling MMIO timers due to CNTFRQ mismatch\n");
1705                         goto out;
1706                 }
1707
1708                 if (!best_frame) /* implies !frame */
1709                         /*
1710                          * Only complain about missing suitable frames if we
1711                          * haven't already found one in a previous iteration.
1712                          */
1713                         pr_err("Unable to find a suitable frame in timer @ %pa\n",
1714                                 &timer->cntctlbase);
1715         }
1716
1717         if (best_frame)
1718                 ret = arch_timer_mem_frame_register(best_frame);
1719 out:
1720         kfree(timers);
1721         return ret;
1722 }
1723
1724 /* Initialize per-processor generic timer and memory-mapped timer(if present) */
1725 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1726 {
1727         int ret, platform_timer_count;
1728
1729         if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1730                 pr_warn("already initialized, skipping\n");
1731                 return -EINVAL;
1732         }
1733
1734         arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1735
1736         ret = acpi_gtdt_init(table, &platform_timer_count);
1737         if (ret)
1738                 return ret;
1739
1740         arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI] =
1741                 acpi_gtdt_map_ppi(ARCH_TIMER_PHYS_NONSECURE_PPI);
1742
1743         arch_timer_ppi[ARCH_TIMER_VIRT_PPI] =
1744                 acpi_gtdt_map_ppi(ARCH_TIMER_VIRT_PPI);
1745
1746         arch_timer_ppi[ARCH_TIMER_HYP_PPI] =
1747                 acpi_gtdt_map_ppi(ARCH_TIMER_HYP_PPI);
1748
1749         arch_timer_populate_kvm_info();
1750
1751         /*
1752          * When probing via ACPI, we have no mechanism to override the sysreg
1753          * CNTFRQ value. This *must* be correct.
1754          */
1755         arch_timer_rate = arch_timer_get_cntfrq();
1756         ret = validate_timer_rate();
1757         if (ret) {
1758                 pr_err(FW_BUG "frequency not available.\n");
1759                 return ret;
1760         }
1761
1762         arch_timer_uses_ppi = arch_timer_select_ppi();
1763         if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1764                 pr_err("No interrupt available, giving up\n");
1765                 return -EINVAL;
1766         }
1767
1768         /* Always-on capability */
1769         arch_timer_c3stop = acpi_gtdt_c3stop(arch_timer_uses_ppi);
1770
1771         /* Check for globally applicable workarounds */
1772         arch_timer_check_ool_workaround(ate_match_acpi_oem_info, table);
1773
1774         ret = arch_timer_register();
1775         if (ret)
1776                 return ret;
1777
1778         if (platform_timer_count &&
1779             arch_timer_mem_acpi_init(platform_timer_count))
1780                 pr_err("Failed to initialize memory-mapped timer.\n");
1781
1782         return arch_timer_common_init();
1783 }
1784 TIMER_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1785 #endif
1786
1787 int kvm_arch_ptp_get_crosststamp(u64 *cycle, struct timespec64 *ts,
1788                                  struct clocksource **cs)
1789 {
1790         struct arm_smccc_res hvc_res;
1791         u32 ptp_counter;
1792         ktime_t ktime;
1793
1794         if (!IS_ENABLED(CONFIG_HAVE_ARM_SMCCC_DISCOVERY))
1795                 return -EOPNOTSUPP;
1796
1797         if (arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI)
1798                 ptp_counter = KVM_PTP_VIRT_COUNTER;
1799         else
1800                 ptp_counter = KVM_PTP_PHYS_COUNTER;
1801
1802         arm_smccc_1_1_invoke(ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID,
1803                              ptp_counter, &hvc_res);
1804
1805         if ((int)(hvc_res.a0) < 0)
1806                 return -EOPNOTSUPP;
1807
1808         ktime = (u64)hvc_res.a0 << 32 | hvc_res.a1;
1809         *ts = ktime_to_timespec64(ktime);
1810         if (cycle)
1811                 *cycle = (u64)hvc_res.a2 << 32 | hvc_res.a3;
1812         if (cs)
1813                 *cs = &clocksource_counter;
1814
1815         return 0;
1816 }
1817 EXPORT_SYMBOL_GPL(kvm_arch_ptp_get_crosststamp);