drivers/clocksource/arm_arch_timer.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  linux/drivers/clocksource/arm_arch_timer.c
   4  *
   5  *  Copyright (C) 2011 ARM Ltd.
   6  *  All Rights Reserved
   7  */
   8
   9 #define pr_fmt(fmt)     "arch_timer: " fmt
  10
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/device.h>
  14 #include <linux/smp.h>
  15 #include <linux/cpu.h>
  16 #include <linux/cpu_pm.h>
  17 #include <linux/clockchips.h>
  18 #include <linux/clocksource.h>
  19 #include <linux/clocksource_ids.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/kstrtox.h>
  22 #include <linux/of_irq.h>
  23 #include <linux/of_address.h>
  24 #include <linux/io.h>
  25 #include <linux/slab.h>
  26 #include <linux/sched/clock.h>
  27 #include <linux/sched_clock.h>
  28 #include <linux/acpi.h>
  29 #include <linux/arm-smccc.h>
  30 #include <linux/ptp_kvm.h>
  31
  32 #include <asm/arch_timer.h>
  33 #include <asm/virt.h>
  34
  35 #include <clocksource/arm_arch_timer.h>
  36
  37 #define CNTTIDR         0x08
  38 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
  39
  40 #define CNTACR(n)       (0x40 + ((n) * 4))
  41 #define CNTACR_RPCT     BIT(0)
  42 #define CNTACR_RVCT     BIT(1)
  43 #define CNTACR_RFRQ     BIT(2)
  44 #define CNTACR_RVOFF    BIT(3)
  45 #define CNTACR_RWVT     BIT(4)
  46 #define CNTACR_RWPT     BIT(5)
  47
  48 #define CNTPCT_LO       0x00
  49 #define CNTVCT_LO       0x08
  50 #define CNTFRQ          0x10
  51 #define CNTP_CVAL_LO    0x20
  52 #define CNTP_CTL        0x2c
  53 #define CNTV_CVAL_LO    0x30
  54 #define CNTV_CTL        0x3c
  55
  56 /*
  57  * The minimum amount of time a generic counter is guaranteed to not roll over
  58  * (40 years)
  59  */
  60 #define MIN_ROLLOVER_SECS       (40ULL * 365 * 24 * 3600)
  61
  62 static unsigned arch_timers_present __initdata;
  63
  64 struct arch_timer {
  65         void __iomem *base;
  66         struct clock_event_device evt;
  67 };
  68
  69 static struct arch_timer *arch_timer_mem __ro_after_init;
  70
  71 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  72
  73 static u32 arch_timer_rate __ro_after_init;
  74 static int arch_timer_ppi[ARCH_TIMER_MAX_TIMER_PPI] __ro_after_init;
  75
  76 static const char *arch_timer_ppi_names[ARCH_TIMER_MAX_TIMER_PPI] = {
  77         [ARCH_TIMER_PHYS_SECURE_PPI]    = "sec-phys",
  78         [ARCH_TIMER_PHYS_NONSECURE_PPI] = "phys",
  79         [ARCH_TIMER_VIRT_PPI]           = "virt",
  80         [ARCH_TIMER_HYP_PPI]            = "hyp-phys",
  81         [ARCH_TIMER_HYP_VIRT_PPI]       = "hyp-virt",
  82 };
  83
  84 static struct clock_event_device __percpu *arch_timer_evt;
  85
  86 static enum arch_timer_ppi_nr arch_timer_uses_ppi __ro_after_init = ARCH_TIMER_VIRT_PPI;
  87 static bool arch_timer_c3stop __ro_after_init;
  88 static bool arch_timer_mem_use_virtual __ro_after_init;
  89 static bool arch_counter_suspend_stop __ro_after_init;
  90 #ifdef CONFIG_GENERIC_GETTIMEOFDAY
  91 static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_ARCHTIMER;
  92 #else
  93 static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_NONE;
  94 #endif /* CONFIG_GENERIC_GETTIMEOFDAY */
  95
  96 static cpumask_t evtstrm_available = CPU_MASK_NONE;
  97 static bool evtstrm_enable __ro_after_init = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
  98
  99 static int __init early_evtstrm_cfg(char *buf)
 100 {
 101         return kstrtobool(buf, &evtstrm_enable);
 102 }
 103 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
 104
 105 /*
 106  * Makes an educated guess at a valid counter width based on the Generic Timer
 107  * specification. Of note:
 108  *   1) the system counter is at least 56 bits wide
 109  *   2) a roll-over time of not less than 40 years
 110  *
 111  * See 'ARM DDI 0487G.a D11.1.2 ("The system counter")' for more details.
 112  */
 113 static int arch_counter_get_width(void)
 114 {
 115         u64 min_cycles = MIN_ROLLOVER_SECS * arch_timer_rate;
 116
 117         /* guarantee the returned width is within the valid range */
 118         return clamp_val(ilog2(min_cycles - 1) + 1, 56, 64);
 119 }
 120
 121 /*
 122  * Architected system timer support.
 123  */
 124
 125 static __always_inline
 126 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u64 val,
 127                           struct clock_event_device *clk)
 128 {
 129         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 130                 struct arch_timer *timer = to_arch_timer(clk);
 131                 switch (reg) {
 132                 case ARCH_TIMER_REG_CTRL:
 133                         writel_relaxed((u32)val, timer->base + CNTP_CTL);
 134                         break;
 135                 case ARCH_TIMER_REG_CVAL:
 136                         /*
 137                          * Not guaranteed to be atomic, so the timer
 138                          * must be disabled at this point.
 139                          */
 140                         writeq_relaxed(val, timer->base + CNTP_CVAL_LO);
 141                         break;
 142                 default:
 143                         BUILD_BUG();
 144                 }
 145         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 146                 struct arch_timer *timer = to_arch_timer(clk);
 147                 switch (reg) {
 148                 case ARCH_TIMER_REG_CTRL:
 149                         writel_relaxed((u32)val, timer->base + CNTV_CTL);
 150                         break;
 151                 case ARCH_TIMER_REG_CVAL:
 152                         /* Same restriction as above */
 153                         writeq_relaxed(val, timer->base + CNTV_CVAL_LO);
 154                         break;
 155                 default:
 156                         BUILD_BUG();
 157                 }
 158         } else {
 159                 arch_timer_reg_write_cp15(access, reg, val);
 160         }
 161 }
 162
 163 static __always_inline
 164 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 165                         struct clock_event_device *clk)
 166 {
 167         u32 val;
 168
 169         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 170                 struct arch_timer *timer = to_arch_timer(clk);
 171                 switch (reg) {
 172                 case ARCH_TIMER_REG_CTRL:
 173                         val = readl_relaxed(timer->base + CNTP_CTL);
 174                         break;
 175                 default:
 176                         BUILD_BUG();
 177                 }
 178         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 179                 struct arch_timer *timer = to_arch_timer(clk);
 180                 switch (reg) {
 181                 case ARCH_TIMER_REG_CTRL:
 182                         val = readl_relaxed(timer->base + CNTV_CTL);
 183                         break;
 184                 default:
 185                         BUILD_BUG();
 186                 }
 187         } else {
 188                 val = arch_timer_reg_read_cp15(access, reg);
 189         }
 190
 191         return val;
 192 }
 193
 194 static noinstr u64 raw_counter_get_cntpct_stable(void)
 195 {
 196         return __arch_counter_get_cntpct_stable();
 197 }
 198
 199 static notrace u64 arch_counter_get_cntpct_stable(void)
 200 {
 201         u64 val;
 202         preempt_disable_notrace();
 203         val = __arch_counter_get_cntpct_stable();
 204         preempt_enable_notrace();
 205         return val;
 206 }
 207
 208 static noinstr u64 arch_counter_get_cntpct(void)
 209 {
 210         return __arch_counter_get_cntpct();
 211 }
 212
 213 static noinstr u64 raw_counter_get_cntvct_stable(void)
 214 {
 215         return __arch_counter_get_cntvct_stable();
 216 }
 217
 218 static notrace u64 arch_counter_get_cntvct_stable(void)
 219 {
 220         u64 val;
 221         preempt_disable_notrace();
 222         val = __arch_counter_get_cntvct_stable();
 223         preempt_enable_notrace();
 224         return val;
 225 }
 226
 227 static noinstr u64 arch_counter_get_cntvct(void)
 228 {
 229         return __arch_counter_get_cntvct();
 230 }
 231
 232 /*
 233  * Default to cp15 based access because arm64 uses this function for
 234  * sched_clock() before DT is probed and the cp15 method is guaranteed
 235  * to exist on arm64. arm doesn't use this before DT is probed so even
 236  * if we don't have the cp15 accessors we won't have a problem.
 237  */
 238 u64 (*arch_timer_read_counter)(void) __ro_after_init = arch_counter_get_cntvct;
 239 EXPORT_SYMBOL_GPL(arch_timer_read_counter);
 240
 241 static u64 arch_counter_read(struct clocksource *cs)
 242 {
 243         return arch_timer_read_counter();
 244 }
 245
 246 static u64 arch_counter_read_cc(const struct cyclecounter *cc)
 247 {
 248         return arch_timer_read_counter();
 249 }
 250
 251 static struct clocksource clocksource_counter = {
 252         .name   = "arch_sys_counter",
 253         .id     = CSID_ARM_ARCH_COUNTER,
 254         .rating = 400,
 255         .read   = arch_counter_read,
 256         .flags  = CLOCK_SOURCE_IS_CONTINUOUS,
 257 };
 258
 259 static struct cyclecounter cyclecounter __ro_after_init = {
 260         .read   = arch_counter_read_cc,
 261 };
 262
 263 struct ate_acpi_oem_info {
 264         char oem_id[ACPI_OEM_ID_SIZE + 1];
 265         char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
 266         u32 oem_revision;
 267 };
 268
 269 #ifdef CONFIG_FSL_ERRATUM_A008585
 270 /*
 271  * The number of retries is an arbitrary value well beyond the highest number
 272  * of iterations the loop has been observed to take.
 273  */
 274 #define __fsl_a008585_read_reg(reg) ({                  \
 275         u64 _old, _new;                                 \
 276         int _retries = 200;                             \
 277                                                         \
 278         do {                                            \
 279                 _old = read_sysreg(reg);                \
 280                 _new = read_sysreg(reg);                \
 281                 _retries--;                             \
 282         } while (unlikely(_old != _new) && _retries);   \
 283                                                         \
 284         WARN_ON_ONCE(!_retries);                        \
 285         _new;                                           \
 286 })
 287
 288 static u64 notrace fsl_a008585_read_cntpct_el0(void)
 289 {
 290         return __fsl_a008585_read_reg(cntpct_el0);
 291 }
 292
 293 static u64 notrace fsl_a008585_read_cntvct_el0(void)
 294 {
 295         return __fsl_a008585_read_reg(cntvct_el0);
 296 }
 297 #endif
 298
 299 #ifdef CONFIG_HISILICON_ERRATUM_161010101
 300 /*
 301  * Verify whether the value of the second read is larger than the first by
 302  * less than 32 is the only way to confirm the value is correct, so clear the
 303  * lower 5 bits to check whether the difference is greater than 32 or not.
 304  * Theoretically the erratum should not occur more than twice in succession
 305  * when reading the system counter, but it is possible that some interrupts
 306  * may lead to more than twice read errors, triggering the warning, so setting
 307  * the number of retries far beyond the number of iterations the loop has been
 308  * observed to take.
 309  */
 310 #define __hisi_161010101_read_reg(reg) ({                               \
 311         u64 _old, _new;                                         \
 312         int _retries = 50;                                      \
 313                                                                 \
 314         do {                                                    \
 315                 _old = read_sysreg(reg);                        \
 316                 _new = read_sysreg(reg);                        \
 317                 _retries--;                                     \
 318         } while (unlikely((_new - _old) >> 5) && _retries);     \
 319                                                                 \
 320         WARN_ON_ONCE(!_retries);                                \
 321         _new;                                                   \
 322 })
 323
 324 static u64 notrace hisi_161010101_read_cntpct_el0(void)
 325 {
 326         return __hisi_161010101_read_reg(cntpct_el0);
 327 }
 328
 329 static u64 notrace hisi_161010101_read_cntvct_el0(void)
 330 {
 331         return __hisi_161010101_read_reg(cntvct_el0);
 332 }
 333
 334 static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {
 335         /*
 336          * Note that trailing spaces are required to properly match
 337          * the OEM table information.
 338          */
 339         {
 340                 .oem_id         = "HISI  ",
 341                 .oem_table_id   = "HIP05   ",
 342                 .oem_revision   = 0,
 343         },
 344         {
 345                 .oem_id         = "HISI  ",
 346                 .oem_table_id   = "HIP06   ",
 347                 .oem_revision   = 0,
 348         },
 349         {
 350                 .oem_id         = "HISI  ",
 351                 .oem_table_id   = "HIP07   ",
 352                 .oem_revision   = 0,
 353         },
 354         { /* Sentinel indicating the end of the OEM array */ },
 355 };
 356 #endif
 357
 358 #ifdef CONFIG_ARM64_ERRATUM_858921
 359 static u64 notrace arm64_858921_read_cntpct_el0(void)
 360 {
 361         u64 old, new;
 362
 363         old = read_sysreg(cntpct_el0);
 364         new = read_sysreg(cntpct_el0);
 365         return (((old ^ new) >> 32) & 1) ? old : new;
 366 }
 367
 368 static u64 notrace arm64_858921_read_cntvct_el0(void)
 369 {
 370         u64 old, new;
 371
 372         old = read_sysreg(cntvct_el0);
 373         new = read_sysreg(cntvct_el0);
 374         return (((old ^ new) >> 32) & 1) ? old : new;
 375 }
 376 #endif
 377
 378 #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
 379 /*
 380  * The low bits of the counter registers are indeterminate while bit 10 or
 381  * greater is rolling over. Since the counter value can jump both backward
 382  * (7ff -> 000 -> 800) and forward (7ff -> fff -> 800), ignore register values
 383  * with all ones or all zeros in the low bits. Bound the loop by the maximum
 384  * number of CPU cycles in 3 consecutive 24 MHz counter periods.
 385  */
 386 #define __sun50i_a64_read_reg(reg) ({                                   \
 387         u64 _val;                                                       \
 388         int _retries = 150;                                             \
 389                                                                         \
 390         do {                                                            \
 391                 _val = read_sysreg(reg);                                \
 392                 _retries--;                                             \
 393         } while (((_val + 1) & GENMASK(8, 0)) <= 1 && _retries);        \
 394                                                                         \
 395         WARN_ON_ONCE(!_retries);                                        \
 396         _val;                                                           \
 397 })
 398
 399 static u64 notrace sun50i_a64_read_cntpct_el0(void)
 400 {
 401         return __sun50i_a64_read_reg(cntpct_el0);
 402 }
 403
 404 static u64 notrace sun50i_a64_read_cntvct_el0(void)
 405 {
 406         return __sun50i_a64_read_reg(cntvct_el0);
 407 }
 408 #endif
 409
 410 #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
 411 DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround);
 412 EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround);
 413
 414 static atomic_t timer_unstable_counter_workaround_in_use = ATOMIC_INIT(0);
 415
 416 /*
 417  * Force the inlining of this function so that the register accesses
 418  * can be themselves correctly inlined.
 419  */
 420 static __always_inline
 421 void erratum_set_next_event_generic(const int access, unsigned long evt,
 422                                     struct clock_event_device *clk)
 423 {
 424         unsigned long ctrl;
 425         u64 cval;
 426
 427         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 428         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 429         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 430
 431         if (access == ARCH_TIMER_PHYS_ACCESS) {
 432                 cval = evt + arch_counter_get_cntpct_stable();
 433                 write_sysreg(cval, cntp_cval_el0);
 434         } else {
 435                 cval = evt + arch_counter_get_cntvct_stable();
 436                 write_sysreg(cval, cntv_cval_el0);
 437         }
 438
 439         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 440 }
 441
 442 static __maybe_unused int erratum_set_next_event_virt(unsigned long evt,
 443                                             struct clock_event_device *clk)
 444 {
 445         erratum_set_next_event_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 446         return 0;
 447 }
 448
 449 static __maybe_unused int erratum_set_next_event_phys(unsigned long evt,
 450                                             struct clock_event_device *clk)
 451 {
 452         erratum_set_next_event_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 453         return 0;
 454 }
 455
 456 static const struct arch_timer_erratum_workaround ool_workarounds[] = {
 457 #ifdef CONFIG_FSL_ERRATUM_A008585
 458         {
 459                 .match_type = ate_match_dt,
 460                 .id = "fsl,erratum-a008585",
 461                 .desc = "Freescale erratum a005858",
 462                 .read_cntpct_el0 = fsl_a008585_read_cntpct_el0,
 463                 .read_cntvct_el0 = fsl_a008585_read_cntvct_el0,
 464                 .set_next_event_phys = erratum_set_next_event_phys,
 465                 .set_next_event_virt = erratum_set_next_event_virt,
 466         },
 467 #endif
 468 #ifdef CONFIG_HISILICON_ERRATUM_161010101
 469         {
 470                 .match_type = ate_match_dt,
 471                 .id = "hisilicon,erratum-161010101",
 472                 .desc = "HiSilicon erratum 161010101",
 473                 .read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
 474                 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
 475                 .set_next_event_phys = erratum_set_next_event_phys,
 476                 .set_next_event_virt = erratum_set_next_event_virt,
 477         },
 478         {
 479                 .match_type = ate_match_acpi_oem_info,
 480                 .id = hisi_161010101_oem_info,
 481                 .desc = "HiSilicon erratum 161010101",
 482                 .read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
 483                 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
 484                 .set_next_event_phys = erratum_set_next_event_phys,
 485                 .set_next_event_virt = erratum_set_next_event_virt,
 486         },
 487 #endif
 488 #ifdef CONFIG_ARM64_ERRATUM_858921
 489         {
 490                 .match_type = ate_match_local_cap_id,
 491                 .id = (void *)ARM64_WORKAROUND_858921,
 492                 .desc = "ARM erratum 858921",
 493                 .read_cntpct_el0 = arm64_858921_read_cntpct_el0,
 494                 .read_cntvct_el0 = arm64_858921_read_cntvct_el0,
 495                 .set_next_event_phys = erratum_set_next_event_phys,
 496                 .set_next_event_virt = erratum_set_next_event_virt,
 497         },
 498 #endif
 499 #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
 500         {
 501                 .match_type = ate_match_dt,
 502                 .id = "allwinner,erratum-unknown1",
 503                 .desc = "Allwinner erratum UNKNOWN1",
 504                 .read_cntpct_el0 = sun50i_a64_read_cntpct_el0,
 505                 .read_cntvct_el0 = sun50i_a64_read_cntvct_el0,
 506                 .set_next_event_phys = erratum_set_next_event_phys,
 507                 .set_next_event_virt = erratum_set_next_event_virt,
 508         },
 509 #endif
 510 #ifdef CONFIG_ARM64_ERRATUM_1418040
 511         {
 512                 .match_type = ate_match_local_cap_id,
 513                 .id = (void *)ARM64_WORKAROUND_1418040,
 514                 .desc = "ARM erratum 1418040",
 515                 .disable_compat_vdso = true,
 516         },
 517 #endif
 518 };
 519
 520 typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,
 521                                const void *);
 522
 523 static
 524 bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa,
 525                                  const void *arg)
 526 {
 527         const struct device_node *np = arg;
 528
 529         return of_property_read_bool(np, wa->id);
 530 }
 531
 532 static
 533 bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa,
 534                                         const void *arg)
 535 {
 536         return this_cpu_has_cap((uintptr_t)wa->id);
 537 }
 538
 539
 540 static
 541 bool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa,
 542                                        const void *arg)
 543 {
 544         static const struct ate_acpi_oem_info empty_oem_info = {};
 545         const struct ate_acpi_oem_info *info = wa->id;
 546         const struct acpi_table_header *table = arg;
 547
 548         /* Iterate over the ACPI OEM info array, looking for a match */
 549         while (memcmp(info, &empty_oem_info, sizeof(*info))) {
 550                 if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) &&
 551                     !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
 552                     info->oem_revision == table->oem_revision)
 553                         return true;
 554
 555                 info++;
 556         }
 557
 558         return false;
 559 }
 560
 561 static const struct arch_timer_erratum_workaround *
 562 arch_timer_iterate_errata(enum arch_timer_erratum_match_type type,
 563                           ate_match_fn_t match_fn,
 564                           void *arg)
 565 {
 566         int i;
 567
 568         for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) {
 569                 if (ool_workarounds[i].match_type != type)
 570                         continue;
 571
 572                 if (match_fn(&ool_workarounds[i], arg))
 573                         return &ool_workarounds[i];
 574         }
 575
 576         return NULL;
 577 }
 578
 579 static
 580 void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa,
 581                                   bool local)
 582 {
 583         int i;
 584
 585         if (local) {
 586                 __this_cpu_write(timer_unstable_counter_workaround, wa);
 587         } else {
 588                 for_each_possible_cpu(i)
 589                         per_cpu(timer_unstable_counter_workaround, i) = wa;
 590         }
 591
 592         if (wa->read_cntvct_el0 || wa->read_cntpct_el0)
 593                 atomic_set(&timer_unstable_counter_workaround_in_use, 1);
 594
 595         /*
 596          * Don't use the vdso fastpath if errata require using the
 597          * out-of-line counter accessor. We may change our mind pretty
 598          * late in the game (with a per-CPU erratum, for example), so
 599          * change both the default value and the vdso itself.
 600          */
 601         if (wa->read_cntvct_el0) {
 602                 clocksource_counter.vdso_clock_mode = VDSO_CLOCKMODE_NONE;
 603                 vdso_default = VDSO_CLOCKMODE_NONE;
 604         } else if (wa->disable_compat_vdso && vdso_default != VDSO_CLOCKMODE_NONE) {
 605                 vdso_default = VDSO_CLOCKMODE_ARCHTIMER_NOCOMPAT;
 606                 clocksource_counter.vdso_clock_mode = vdso_default;
 607         }
 608 }
 609
 610 static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,
 611                                             void *arg)
 612 {
 613         const struct arch_timer_erratum_workaround *wa, *__wa;
 614         ate_match_fn_t match_fn = NULL;
 615         bool local = false;
 616
 617         switch (type) {
 618         case ate_match_dt:
 619                 match_fn = arch_timer_check_dt_erratum;
 620                 break;
 621         case ate_match_local_cap_id:
 622                 match_fn = arch_timer_check_local_cap_erratum;
 623                 local = true;
 624                 break;
 625         case ate_match_acpi_oem_info:
 626                 match_fn = arch_timer_check_acpi_oem_erratum;
 627                 break;
 628         default:
 629                 WARN_ON(1);
 630                 return;
 631         }
 632
 633         wa = arch_timer_iterate_errata(type, match_fn, arg);
 634         if (!wa)
 635                 return;
 636
 637         __wa = __this_cpu_read(timer_unstable_counter_workaround);
 638         if (__wa && wa != __wa)
 639                 pr_warn("Can't enable workaround for %s (clashes with %s\n)",
 640                         wa->desc, __wa->desc);
 641
 642         if (__wa)
 643                 return;
 644
 645         arch_timer_enable_workaround(wa, local);
 646         pr_info("Enabling %s workaround for %s\n",
 647                 local ? "local" : "global", wa->desc);
 648 }
 649
 650 static bool arch_timer_this_cpu_has_cntvct_wa(void)
 651 {
 652         return has_erratum_handler(read_cntvct_el0);
 653 }
 654
 655 static bool arch_timer_counter_has_wa(void)
 656 {
 657         return atomic_read(&timer_unstable_counter_workaround_in_use);
 658 }
 659 #else
 660 #define arch_timer_check_ool_workaround(t,a)            do { } while(0)
 661 #define arch_timer_this_cpu_has_cntvct_wa()             ({false;})
 662 #define arch_timer_counter_has_wa()                     ({false;})
 663 #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
 664
 665 static __always_inline irqreturn_t timer_handler(const int access,
 666                                         struct clock_event_device *evt)
 667 {
 668         unsigned long ctrl;
 669
 670         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 671         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 672                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 673                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 674                 evt->event_handler(evt);
 675                 return IRQ_HANDLED;
 676         }
 677
 678         return IRQ_NONE;
 679 }
 680
 681 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 682 {
 683         struct clock_event_device *evt = dev_id;
 684
 685         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 686 }
 687
 688 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 689 {
 690         struct clock_event_device *evt = dev_id;
 691
 692         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 693 }
 694
 695 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 696 {
 697         struct clock_event_device *evt = dev_id;
 698
 699         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 700 }
 701
 702 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 703 {
 704         struct clock_event_device *evt = dev_id;
 705
 706         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 707 }
 708
 709 static __always_inline int arch_timer_shutdown(const int access,
 710                                                struct clock_event_device *clk)
 711 {
 712         unsigned long ctrl;
 713
 714         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 715         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 716         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 717
 718         return 0;
 719 }
 720
 721 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
 722 {
 723         return arch_timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
 724 }
 725
 726 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
 727 {
 728         return arch_timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
 729 }
 730
 731 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
 732 {
 733         return arch_timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
 734 }
 735
 736 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
 737 {
 738         return arch_timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
 739 }
 740
 741 static __always_inline void set_next_event(const int access, unsigned long evt,
 742                                            struct clock_event_device *clk)
 743 {
 744         unsigned long ctrl;
 745         u64 cnt;
 746
 747         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 748         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 749         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 750
 751         if (access == ARCH_TIMER_PHYS_ACCESS)
 752                 cnt = __arch_counter_get_cntpct();
 753         else
 754                 cnt = __arch_counter_get_cntvct();
 755
 756         arch_timer_reg_write(access, ARCH_TIMER_REG_CVAL, evt + cnt, clk);
 757         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 758 }
 759
 760 static int arch_timer_set_next_event_virt(unsigned long evt,
 761                                           struct clock_event_device *clk)
 762 {
 763         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 764         return 0;
 765 }
 766
 767 static int arch_timer_set_next_event_phys(unsigned long evt,
 768                                           struct clock_event_device *clk)
 769 {
 770         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 771         return 0;
 772 }
 773
 774 static noinstr u64 arch_counter_get_cnt_mem(struct arch_timer *t, int offset_lo)
 775 {
 776         u32 cnt_lo, cnt_hi, tmp_hi;
 777
 778         do {
 779                 cnt_hi = __le32_to_cpu((__le32 __force)__raw_readl(t->base + offset_lo + 4));
 780                 cnt_lo = __le32_to_cpu((__le32 __force)__raw_readl(t->base + offset_lo));
 781                 tmp_hi = __le32_to_cpu((__le32 __force)__raw_readl(t->base + offset_lo + 4));
 782         } while (cnt_hi != tmp_hi);
 783
 784         return ((u64) cnt_hi << 32) | cnt_lo;
 785 }
 786
 787 static __always_inline void set_next_event_mem(const int access, unsigned long evt,
 788                                            struct clock_event_device *clk)
 789 {
 790         struct arch_timer *timer = to_arch_timer(clk);
 791         unsigned long ctrl;
 792         u64 cnt;
 793
 794         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 795
 796         /* Timer must be disabled before programming CVAL */
 797         if (ctrl & ARCH_TIMER_CTRL_ENABLE) {
 798                 ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 799                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 800         }
 801
 802         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 803         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 804
 805         if (access ==  ARCH_TIMER_MEM_VIRT_ACCESS)
 806                 cnt = arch_counter_get_cnt_mem(timer, CNTVCT_LO);
 807         else
 808                 cnt = arch_counter_get_cnt_mem(timer, CNTPCT_LO);
 809
 810         arch_timer_reg_write(access, ARCH_TIMER_REG_CVAL, evt + cnt, clk);
 811         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 812 }
 813
 814 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 815                                               struct clock_event_device *clk)
 816 {
 817         set_next_event_mem(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 818         return 0;
 819 }
 820
 821 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 822                                               struct clock_event_device *clk)
 823 {
 824         set_next_event_mem(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 825         return 0;
 826 }
 827
 828 static u64 __arch_timer_check_delta(void)
 829 {
 830 #ifdef CONFIG_ARM64
 831         const struct midr_range broken_cval_midrs[] = {
 832                 /*
 833                  * XGene-1 implements CVAL in terms of TVAL, meaning
 834                  * that the maximum timer range is 32bit. Shame on them.
 835                  *
 836                  * Note that TVAL is signed, thus has only 31 of its
 837                  * 32 bits to express magnitude.
 838                  */
 839                 MIDR_REV_RANGE(MIDR_CPU_MODEL(ARM_CPU_IMP_APM,
 840                                               APM_CPU_PART_XGENE),
 841                                APM_CPU_VAR_POTENZA, 0x0, 0xf),
 842                 {},
 843         };
 844
 845         if (is_midr_in_range_list(read_cpuid_id(), broken_cval_midrs)) {
 846                 pr_warn_once("Broken CNTx_CVAL_EL1, using 31 bit TVAL instead.\n");
 847                 return CLOCKSOURCE_MASK(31);
 848         }
 849 #endif
 850         return CLOCKSOURCE_MASK(arch_counter_get_width());
 851 }
 852
 853 static void __arch_timer_setup(unsigned type,
 854                                struct clock_event_device *clk)
 855 {
 856         u64 max_delta;
 857
 858         clk->features = CLOCK_EVT_FEAT_ONESHOT;
 859
 860         if (type == ARCH_TIMER_TYPE_CP15) {
 861                 typeof(clk->set_next_event) sne;
 862
 863                 arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);
 864
 865                 if (arch_timer_c3stop)
 866                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
 867                 clk->name = "arch_sys_timer";
 868                 clk->rating = 450;
 869                 clk->cpumask = cpumask_of(smp_processor_id());
 870                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
 871                 switch (arch_timer_uses_ppi) {
 872                 case ARCH_TIMER_VIRT_PPI:
 873                         clk->set_state_shutdown = arch_timer_shutdown_virt;
 874                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
 875                         sne = erratum_handler(set_next_event_virt);
 876                         break;
 877                 case ARCH_TIMER_PHYS_SECURE_PPI:
 878                 case ARCH_TIMER_PHYS_NONSECURE_PPI:
 879                 case ARCH_TIMER_HYP_PPI:
 880                         clk->set_state_shutdown = arch_timer_shutdown_phys;
 881                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
 882                         sne = erratum_handler(set_next_event_phys);
 883                         break;
 884                 default:
 885                         BUG();
 886                 }
 887
 888                 clk->set_next_event = sne;
 889                 max_delta = __arch_timer_check_delta();
 890         } else {
 891                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 892                 clk->name = "arch_mem_timer";
 893                 clk->rating = 400;
 894                 clk->cpumask = cpu_possible_mask;
 895                 if (arch_timer_mem_use_virtual) {
 896                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
 897                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
 898                         clk->set_next_event =
 899                                 arch_timer_set_next_event_virt_mem;
 900                 } else {
 901                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
 902                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
 903                         clk->set_next_event =
 904                                 arch_timer_set_next_event_phys_mem;
 905                 }
 906
 907                 max_delta = CLOCKSOURCE_MASK(56);
 908         }
 909
 910         clk->set_state_shutdown(clk);
 911
 912         clockevents_config_and_register(clk, arch_timer_rate, 0xf, max_delta);
 913 }
 914
 915 static void arch_timer_evtstrm_enable(unsigned int divider)
 916 {
 917         u32 cntkctl = arch_timer_get_cntkctl();
 918
 919 #ifdef CONFIG_ARM64
 920         /* ECV is likely to require a large divider. Use the EVNTIS flag. */
 921         if (cpus_have_const_cap(ARM64_HAS_ECV) && divider > 15) {
 922                 cntkctl |= ARCH_TIMER_EVT_INTERVAL_SCALE;
 923                 divider -= 8;
 924         }
 925 #endif
 926
 927         divider = min(divider, 15U);
 928         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 929         /* Set the divider and enable virtual event stream */
 930         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 931                         | ARCH_TIMER_VIRT_EVT_EN;
 932         arch_timer_set_cntkctl(cntkctl);
 933         arch_timer_set_evtstrm_feature();
 934         cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
 935 }
 936
 937 static void arch_timer_configure_evtstream(void)
 938 {
 939         int evt_stream_div, lsb;
 940
 941         /*
 942          * As the event stream can at most be generated at half the frequency
 943          * of the counter, use half the frequency when computing the divider.
 944          */
 945         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ / 2;
 946
 947         /*
 948          * Find the closest power of two to the divisor. If the adjacent bit
 949          * of lsb (last set bit, starts from 0) is set, then we use (lsb + 1).
 950          */
 951         lsb = fls(evt_stream_div) - 1;
 952         if (lsb > 0 && (evt_stream_div & BIT(lsb - 1)))
 953                 lsb++;
 954
 955         /* enable event stream */
 956         arch_timer_evtstrm_enable(max(0, lsb));
 957 }
 958
 959 static void arch_counter_set_user_access(void)
 960 {
 961         u32 cntkctl = arch_timer_get_cntkctl();
 962
 963         /* Disable user access to the timers and both counters */
 964         /* Also disable virtual event stream */
 965         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 966                         | ARCH_TIMER_USR_VT_ACCESS_EN
 967                         | ARCH_TIMER_USR_VCT_ACCESS_EN
 968                         | ARCH_TIMER_VIRT_EVT_EN
 969                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
 970
 971         /*
 972          * Enable user access to the virtual counter if it doesn't
 973          * need to be workaround. The vdso may have been already
 974          * disabled though.
 975          */
 976         if (arch_timer_this_cpu_has_cntvct_wa())
 977                 pr_info("CPU%d: Trapping CNTVCT access\n", smp_processor_id());
 978         else
 979                 cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 980
 981         arch_timer_set_cntkctl(cntkctl);
 982 }
 983
 984 static bool arch_timer_has_nonsecure_ppi(void)
 985 {
 986         return (arch_timer_uses_ppi == ARCH_TIMER_PHYS_SECURE_PPI &&
 987                 arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
 988 }
 989
 990 static u32 check_ppi_trigger(int irq)
 991 {
 992         u32 flags = irq_get_trigger_type(irq);
 993
 994         if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
 995                 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
 996                 pr_warn("WARNING: Please fix your firmware\n");
 997                 flags = IRQF_TRIGGER_LOW;
 998         }
 999
1000         return flags;
1001 }
1002
1003 static int arch_timer_starting_cpu(unsigned int cpu)
1004 {
1005         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
1006         u32 flags;
1007
1008         __arch_timer_setup(ARCH_TIMER_TYPE_CP15, clk);
1009
1010         flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
1011         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
1012
1013         if (arch_timer_has_nonsecure_ppi()) {
1014                 flags = check_ppi_trigger(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
1015                 enable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
1016                                   flags);
1017         }
1018
1019         arch_counter_set_user_access();
1020         if (evtstrm_enable)
1021                 arch_timer_configure_evtstream();
1022
1023         return 0;
1024 }
1025
1026 static int validate_timer_rate(void)
1027 {
1028         if (!arch_timer_rate)
1029                 return -EINVAL;
1030
1031         /* Arch timer frequency < 1MHz can cause trouble */
1032         WARN_ON(arch_timer_rate < 1000000);
1033
1034         return 0;
1035 }
1036
1037 /*
1038  * For historical reasons, when probing with DT we use whichever (non-zero)
1039  * rate was probed first, and don't verify that others match. If the first node
1040  * probed has a clock-frequency property, this overrides the HW register.
1041  */
1042 static void __init arch_timer_of_configure_rate(u32 rate, struct device_node *np)
1043 {
1044         /* Who has more than one independent system counter? */
1045         if (arch_timer_rate)
1046                 return;
1047
1048         if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate))
1049                 arch_timer_rate = rate;
1050
1051         /* Check the timer frequency. */
1052         if (validate_timer_rate())
1053                 pr_warn("frequency not available\n");
1054 }
1055
1056 static void __init arch_timer_banner(unsigned type)
1057 {
1058         pr_info("%s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
1059                 type & ARCH_TIMER_TYPE_CP15 ? "cp15" : "",
1060                 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ?
1061                         " and " : "",
1062                 type & ARCH_TIMER_TYPE_MEM ? "mmio" : "",
1063                 (unsigned long)arch_timer_rate / 1000000,
1064                 (unsigned long)(arch_timer_rate / 10000) % 100,
1065                 type & ARCH_TIMER_TYPE_CP15 ?
1066                         (arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) ? "virt" : "phys" :
1067                         "",
1068                 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? "/" : "",
1069                 type & ARCH_TIMER_TYPE_MEM ?
1070                         arch_timer_mem_use_virtual ? "virt" : "phys" :
1071                         "");
1072 }
1073
1074 u32 arch_timer_get_rate(void)
1075 {
1076         return arch_timer_rate;
1077 }
1078
1079 bool arch_timer_evtstrm_available(void)
1080 {
1081         /*
1082          * We might get called from a preemptible context. This is fine
1083          * because availability of the event stream should be always the same
1084          * for a preemptible context and context where we might resume a task.
1085          */
1086         return cpumask_test_cpu(raw_smp_processor_id(), &evtstrm_available);
1087 }
1088
1089 static noinstr u64 arch_counter_get_cntvct_mem(void)
1090 {
1091         return arch_counter_get_cnt_mem(arch_timer_mem, CNTVCT_LO);
1092 }
1093
1094 static struct arch_timer_kvm_info arch_timer_kvm_info;
1095
1096 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
1097 {
1098         return &arch_timer_kvm_info;
1099 }
1100
1101 static void __init arch_counter_register(unsigned type)
1102 {
1103         u64 (*scr)(void);
1104         u64 start_count;
1105         int width;
1106
1107         /* Register the CP15 based counter if we have one */
1108         if (type & ARCH_TIMER_TYPE_CP15) {
1109                 u64 (*rd)(void);
1110
1111                 if ((IS_ENABLED(CONFIG_ARM64) && !is_hyp_mode_available()) ||
1112                     arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) {
1113                         if (arch_timer_counter_has_wa()) {
1114                                 rd = arch_counter_get_cntvct_stable;
1115                                 scr = raw_counter_get_cntvct_stable;
1116                         } else {
1117                                 rd = arch_counter_get_cntvct;
1118                                 scr = arch_counter_get_cntvct;
1119                         }
1120                 } else {
1121                         if (arch_timer_counter_has_wa()) {
1122                                 rd = arch_counter_get_cntpct_stable;
1123                                 scr = raw_counter_get_cntpct_stable;
1124                         } else {
1125                                 rd = arch_counter_get_cntpct;
1126                                 scr = arch_counter_get_cntpct;
1127                         }
1128                 }
1129
1130                 arch_timer_read_counter = rd;
1131                 clocksource_counter.vdso_clock_mode = vdso_default;
1132         } else {
1133                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
1134                 scr = arch_counter_get_cntvct_mem;
1135         }
1136
1137         width = arch_counter_get_width();
1138         clocksource_counter.mask = CLOCKSOURCE_MASK(width);
1139         cyclecounter.mask = CLOCKSOURCE_MASK(width);
1140
1141         if (!arch_counter_suspend_stop)
1142                 clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
1143         start_count = arch_timer_read_counter();
1144         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
1145         cyclecounter.mult = clocksource_counter.mult;
1146         cyclecounter.shift = clocksource_counter.shift;
1147         timecounter_init(&arch_timer_kvm_info.timecounter,
1148                          &cyclecounter, start_count);
1149
1150         sched_clock_register(scr, width, arch_timer_rate);
1151 }
1152
1153 static void arch_timer_stop(struct clock_event_device *clk)
1154 {
1155         pr_debug("disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id());
1156
1157         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
1158         if (arch_timer_has_nonsecure_ppi())
1159                 disable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
1160
1161         clk->set_state_shutdown(clk);
1162 }
1163
1164 static int arch_timer_dying_cpu(unsigned int cpu)
1165 {
1166         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
1167
1168         cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1169
1170         arch_timer_stop(clk);
1171         return 0;
1172 }
1173
1174 #ifdef CONFIG_CPU_PM
1175 static DEFINE_PER_CPU(unsigned long, saved_cntkctl);
1176 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
1177                                     unsigned long action, void *hcpu)
1178 {
1179         if (action == CPU_PM_ENTER) {
1180                 __this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl());
1181
1182                 cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1183         } else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) {
1184                 arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
1185
1186                 if (arch_timer_have_evtstrm_feature())
1187                         cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
1188         }
1189         return NOTIFY_OK;
1190 }
1191
1192 static struct notifier_block arch_timer_cpu_pm_notifier = {
1193         .notifier_call = arch_timer_cpu_pm_notify,
1194 };
1195
1196 static int __init arch_timer_cpu_pm_init(void)
1197 {
1198         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
1199 }
1200
1201 static void __init arch_timer_cpu_pm_deinit(void)
1202 {
1203         WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
1204 }
1205
1206 #else
1207 static int __init arch_timer_cpu_pm_init(void)
1208 {
1209         return 0;
1210 }
1211
1212 static void __init arch_timer_cpu_pm_deinit(void)
1213 {
1214 }
1215 #endif
1216
1217 static int __init arch_timer_register(void)
1218 {
1219         int err;
1220         int ppi;
1221
1222         arch_timer_evt = alloc_percpu(struct clock_event_device);
1223         if (!arch_timer_evt) {
1224                 err = -ENOMEM;
1225                 goto out;
1226         }
1227
1228         ppi = arch_timer_ppi[arch_timer_uses_ppi];
1229         switch (arch_timer_uses_ppi) {
1230         case ARCH_TIMER_VIRT_PPI:
1231                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
1232                                          "arch_timer", arch_timer_evt);
1233                 break;
1234         case ARCH_TIMER_PHYS_SECURE_PPI:
1235         case ARCH_TIMER_PHYS_NONSECURE_PPI:
1236                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
1237                                          "arch_timer", arch_timer_evt);
1238                 if (!err && arch_timer_has_nonsecure_ppi()) {
1239                         ppi = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1240                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
1241                                                  "arch_timer", arch_timer_evt);
1242                         if (err)
1243                                 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_SECURE_PPI],
1244                                                 arch_timer_evt);
1245                 }
1246                 break;
1247         case ARCH_TIMER_HYP_PPI:
1248                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
1249                                          "arch_timer", arch_timer_evt);
1250                 break;
1251         default:
1252                 BUG();
1253         }
1254
1255         if (err) {
1256                 pr_err("can't register interrupt %d (%d)\n", ppi, err);
1257                 goto out_free;
1258         }
1259
1260         err = arch_timer_cpu_pm_init();
1261         if (err)
1262                 goto out_unreg_notify;
1263
1264         /* Register and immediately configure the timer on the boot CPU */
1265         err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
1266                                 "clockevents/arm/arch_timer:starting",
1267                                 arch_timer_starting_cpu, arch_timer_dying_cpu);
1268         if (err)
1269                 goto out_unreg_cpupm;
1270         return 0;
1271
1272 out_unreg_cpupm:
1273         arch_timer_cpu_pm_deinit();
1274
1275 out_unreg_notify:
1276         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
1277         if (arch_timer_has_nonsecure_ppi())
1278                 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
1279                                 arch_timer_evt);
1280
1281 out_free:
1282         free_percpu(arch_timer_evt);
1283 out:
1284         return err;
1285 }
1286
1287 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
1288 {
1289         int ret;
1290         irq_handler_t func;
1291
1292         arch_timer_mem = kzalloc(sizeof(*arch_timer_mem), GFP_KERNEL);
1293         if (!arch_timer_mem)
1294                 return -ENOMEM;
1295
1296         arch_timer_mem->base = base;
1297         arch_timer_mem->evt.irq = irq;
1298         __arch_timer_setup(ARCH_TIMER_TYPE_MEM, &arch_timer_mem->evt);
1299
1300         if (arch_timer_mem_use_virtual)
1301                 func = arch_timer_handler_virt_mem;
1302         else
1303                 func = arch_timer_handler_phys_mem;
1304
1305         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &arch_timer_mem->evt);
1306         if (ret) {
1307                 pr_err("Failed to request mem timer irq\n");
1308                 kfree(arch_timer_mem);
1309                 arch_timer_mem = NULL;
1310         }
1311
1312         return ret;
1313 }
1314
1315 static const struct of_device_id arch_timer_of_match[] __initconst = {
1316         { .compatible   = "arm,armv7-timer",    },
1317         { .compatible   = "arm,armv8-timer",    },
1318         {},
1319 };
1320
1321 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
1322         { .compatible   = "arm,armv7-timer-mem", },
1323         {},
1324 };
1325
1326 static bool __init arch_timer_needs_of_probing(void)
1327 {
1328         struct device_node *dn;
1329         bool needs_probing = false;
1330         unsigned int mask = ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM;
1331
1332         /* We have two timers, and both device-tree nodes are probed. */
1333         if ((arch_timers_present & mask) == mask)
1334                 return false;
1335
1336         /*
1337          * Only one type of timer is probed,
1338          * check if we have another type of timer node in device-tree.
1339          */
1340         if (arch_timers_present & ARCH_TIMER_TYPE_CP15)
1341                 dn = of_find_matching_node(NULL, arch_timer_mem_of_match);
1342         else
1343                 dn = of_find_matching_node(NULL, arch_timer_of_match);
1344
1345         if (dn && of_device_is_available(dn))
1346                 needs_probing = true;
1347
1348         of_node_put(dn);
1349
1350         return needs_probing;
1351 }
1352
1353 static int __init arch_timer_common_init(void)
1354 {
1355         arch_timer_banner(arch_timers_present);
1356         arch_counter_register(arch_timers_present);
1357         return arch_timer_arch_init();
1358 }
1359
1360 /**
1361  * arch_timer_select_ppi() - Select suitable PPI for the current system.
1362  *
1363  * If HYP mode is available, we know that the physical timer
1364  * has been configured to be accessible from PL1. Use it, so
1365  * that a guest can use the virtual timer instead.
1366  *
1367  * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
1368  * accesses to CNTP_*_EL1 registers are silently redirected to
1369  * their CNTHP_*_EL2 counterparts, and use a different PPI
1370  * number.
1371  *
1372  * If no interrupt provided for virtual timer, we'll have to
1373  * stick to the physical timer. It'd better be accessible...
1374  * For arm64 we never use the secure interrupt.
1375  *
1376  * Return: a suitable PPI type for the current system.
1377  */
1378 static enum arch_timer_ppi_nr __init arch_timer_select_ppi(void)
1379 {
1380         if (is_kernel_in_hyp_mode())
1381                 return ARCH_TIMER_HYP_PPI;
1382
1383         if (!is_hyp_mode_available() && arch_timer_ppi[ARCH_TIMER_VIRT_PPI])
1384                 return ARCH_TIMER_VIRT_PPI;
1385
1386         if (IS_ENABLED(CONFIG_ARM64))
1387                 return ARCH_TIMER_PHYS_NONSECURE_PPI;
1388
1389         return ARCH_TIMER_PHYS_SECURE_PPI;
1390 }
1391
1392 static void __init arch_timer_populate_kvm_info(void)
1393 {
1394         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1395         if (is_kernel_in_hyp_mode())
1396                 arch_timer_kvm_info.physical_irq = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1397 }
1398
1399 static int __init arch_timer_of_init(struct device_node *np)
1400 {
1401         int i, irq, ret;
1402         u32 rate;
1403         bool has_names;
1404
1405         if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1406                 pr_warn("multiple nodes in dt, skipping\n");
1407                 return 0;
1408         }
1409
1410         arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1411
1412         has_names = of_property_read_bool(np, "interrupt-names");
1413
1414         for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++) {
1415                 if (has_names)
1416                         irq = of_irq_get_byname(np, arch_timer_ppi_names[i]);
1417                 else
1418                         irq = of_irq_get(np, i);
1419                 if (irq > 0)
1420                         arch_timer_ppi[i] = irq;
1421         }
1422
1423         arch_timer_populate_kvm_info();
1424
1425         rate = arch_timer_get_cntfrq();
1426         arch_timer_of_configure_rate(rate, np);
1427
1428         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
1429
1430         /* Check for globally applicable workarounds */
1431         arch_timer_check_ool_workaround(ate_match_dt, np);
1432
1433         /*
1434          * If we cannot rely on firmware initializing the timer registers then
1435          * we should use the physical timers instead.
1436          */
1437         if (IS_ENABLED(CONFIG_ARM) &&
1438             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
1439                 arch_timer_uses_ppi = ARCH_TIMER_PHYS_SECURE_PPI;
1440         else
1441                 arch_timer_uses_ppi = arch_timer_select_ppi();
1442
1443         if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1444                 pr_err("No interrupt available, giving up\n");
1445                 return -EINVAL;
1446         }
1447
1448         /* On some systems, the counter stops ticking when in suspend. */
1449         arch_counter_suspend_stop = of_property_read_bool(np,
1450                                                          "arm,no-tick-in-suspend");
1451
1452         ret = arch_timer_register();
1453         if (ret)
1454                 return ret;
1455
1456         if (arch_timer_needs_of_probing())
1457                 return 0;
1458
1459         return arch_timer_common_init();
1460 }
1461 TIMER_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
1462 TIMER_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
1463
1464 static u32 __init
1465 arch_timer_mem_frame_get_cntfrq(struct arch_timer_mem_frame *frame)
1466 {
1467         void __iomem *base;
1468         u32 rate;
1469
1470         base = ioremap(frame->cntbase, frame->size);
1471         if (!base) {
1472                 pr_err("Unable to map frame @ %pa\n", &frame->cntbase);
1473                 return 0;
1474         }
1475
1476         rate = readl_relaxed(base + CNTFRQ);
1477
1478         iounmap(base);
1479
1480         return rate;
1481 }
1482
1483 static struct arch_timer_mem_frame * __init
1484 arch_timer_mem_find_best_frame(struct arch_timer_mem *timer_mem)
1485 {
1486         struct arch_timer_mem_frame *frame, *best_frame = NULL;
1487         void __iomem *cntctlbase;
1488         u32 cnttidr;
1489         int i;
1490
1491         cntctlbase = ioremap(timer_mem->cntctlbase, timer_mem->size);
1492         if (!cntctlbase) {
1493                 pr_err("Can't map CNTCTLBase @ %pa\n",
1494                         &timer_mem->cntctlbase);
1495                 return NULL;
1496         }
1497
1498         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
1499
1500         /*
1501          * Try to find a virtual capable frame. Otherwise fall back to a
1502          * physical capable frame.
1503          */
1504         for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1505                 u32 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
1506                              CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
1507
1508                 frame = &timer_mem->frame[i];
1509                 if (!frame->valid)
1510                         continue;
1511
1512                 /* Try enabling everything, and see what sticks */
1513                 writel_relaxed(cntacr, cntctlbase + CNTACR(i));
1514                 cntacr = readl_relaxed(cntctlbase + CNTACR(i));
1515
1516                 if ((cnttidr & CNTTIDR_VIRT(i)) &&
1517                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
1518                         best_frame = frame;
1519                         arch_timer_mem_use_virtual = true;
1520                         break;
1521                 }
1522
1523                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
1524                         continue;
1525
1526                 best_frame = frame;
1527         }
1528
1529         iounmap(cntctlbase);
1530
1531         return best_frame;
1532 }
1533
1534 static int __init
1535 arch_timer_mem_frame_register(struct arch_timer_mem_frame *frame)
1536 {
1537         void __iomem *base;
1538         int ret, irq = 0;
1539
1540         if (arch_timer_mem_use_virtual)
1541                 irq = frame->virt_irq;
1542         else
1543                 irq = frame->phys_irq;
1544
1545         if (!irq) {
1546                 pr_err("Frame missing %s irq.\n",
1547                        arch_timer_mem_use_virtual ? "virt" : "phys");
1548                 return -EINVAL;
1549         }
1550
1551         if (!request_mem_region(frame->cntbase, frame->size,
1552                                 "arch_mem_timer"))
1553                 return -EBUSY;
1554
1555         base = ioremap(frame->cntbase, frame->size);
1556         if (!base) {
1557                 pr_err("Can't map frame's registers\n");
1558                 return -ENXIO;
1559         }
1560
1561         ret = arch_timer_mem_register(base, irq);
1562         if (ret) {
1563                 iounmap(base);
1564                 return ret;
1565         }
1566
1567         arch_timers_present |= ARCH_TIMER_TYPE_MEM;
1568
1569         return 0;
1570 }
1571
1572 static int __init arch_timer_mem_of_init(struct device_node *np)
1573 {
1574         struct arch_timer_mem *timer_mem;
1575         struct arch_timer_mem_frame *frame;
1576         struct device_node *frame_node;
1577         struct resource res;
1578         int ret = -EINVAL;
1579         u32 rate;
1580
1581         timer_mem = kzalloc(sizeof(*timer_mem), GFP_KERNEL);
1582         if (!timer_mem)
1583                 return -ENOMEM;
1584
1585         if (of_address_to_resource(np, 0, &res))
1586                 goto out;
1587         timer_mem->cntctlbase = res.start;
1588         timer_mem->size = resource_size(&res);
1589
1590         for_each_available_child_of_node(np, frame_node) {
1591                 u32 n;
1592                 struct arch_timer_mem_frame *frame;
1593
1594                 if (of_property_read_u32(frame_node, "frame-number", &n)) {
1595                         pr_err(FW_BUG "Missing frame-number.\n");
1596                         of_node_put(frame_node);
1597                         goto out;
1598                 }
1599                 if (n >= ARCH_TIMER_MEM_MAX_FRAMES) {
1600                         pr_err(FW_BUG "Wrong frame-number, only 0-%u are permitted.\n",
1601                                ARCH_TIMER_MEM_MAX_FRAMES - 1);
1602                         of_node_put(frame_node);
1603                         goto out;
1604                 }
1605                 frame = &timer_mem->frame[n];
1606
1607                 if (frame->valid) {
1608                         pr_err(FW_BUG "Duplicated frame-number.\n");
1609                         of_node_put(frame_node);
1610                         goto out;
1611                 }
1612
1613                 if (of_address_to_resource(frame_node, 0, &res)) {
1614                         of_node_put(frame_node);
1615                         goto out;
1616                 }
1617                 frame->cntbase = res.start;
1618                 frame->size = resource_size(&res);
1619
1620                 frame->virt_irq = irq_of_parse_and_map(frame_node,
1621                                                        ARCH_TIMER_VIRT_SPI);
1622                 frame->phys_irq = irq_of_parse_and_map(frame_node,
1623                                                        ARCH_TIMER_PHYS_SPI);
1624
1625                 frame->valid = true;
1626         }
1627
1628         frame = arch_timer_mem_find_best_frame(timer_mem);
1629         if (!frame) {
1630                 pr_err("Unable to find a suitable frame in timer @ %pa\n",
1631                         &timer_mem->cntctlbase);
1632                 ret = -EINVAL;
1633                 goto out;
1634         }
1635
1636         rate = arch_timer_mem_frame_get_cntfrq(frame);
1637         arch_timer_of_configure_rate(rate, np);
1638
1639         ret = arch_timer_mem_frame_register(frame);
1640         if (!ret && !arch_timer_needs_of_probing())
1641                 ret = arch_timer_common_init();
1642 out:
1643         kfree(timer_mem);
1644         return ret;
1645 }
1646 TIMER_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
1647                        arch_timer_mem_of_init);
1648
1649 #ifdef CONFIG_ACPI_GTDT
1650 static int __init
1651 arch_timer_mem_verify_cntfrq(struct arch_timer_mem *timer_mem)
1652 {
1653         struct arch_timer_mem_frame *frame;
1654         u32 rate;
1655         int i;
1656
1657         for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1658                 frame = &timer_mem->frame[i];
1659
1660                 if (!frame->valid)
1661                         continue;
1662
1663                 rate = arch_timer_mem_frame_get_cntfrq(frame);
1664                 if (rate == arch_timer_rate)
1665                         continue;
1666
1667                 pr_err(FW_BUG "CNTFRQ mismatch: frame @ %pa: (0x%08lx), CPU: (0x%08lx)\n",
1668                         &frame->cntbase,
1669                         (unsigned long)rate, (unsigned long)arch_timer_rate);
1670
1671                 return -EINVAL;
1672         }
1673
1674         return 0;
1675 }
1676
1677 static int __init arch_timer_mem_acpi_init(int platform_timer_count)
1678 {
1679         struct arch_timer_mem *timers, *timer;
1680         struct arch_timer_mem_frame *frame, *best_frame = NULL;
1681         int timer_count, i, ret = 0;
1682
1683         timers = kcalloc(platform_timer_count, sizeof(*timers),
1684                             GFP_KERNEL);
1685         if (!timers)
1686                 return -ENOMEM;
1687
1688         ret = acpi_arch_timer_mem_init(timers, &timer_count);
1689         if (ret || !timer_count)
1690                 goto out;
1691
1692         /*
1693          * While unlikely, it's theoretically possible that none of the frames
1694          * in a timer expose the combination of feature we want.
1695          */
1696         for (i = 0; i < timer_count; i++) {
1697                 timer = &timers[i];
1698
1699                 frame = arch_timer_mem_find_best_frame(timer);
1700                 if (!best_frame)
1701                         best_frame = frame;
1702
1703                 ret = arch_timer_mem_verify_cntfrq(timer);
1704                 if (ret) {
1705                         pr_err("Disabling MMIO timers due to CNTFRQ mismatch\n");
1706                         goto out;
1707                 }
1708
1709                 if (!best_frame) /* implies !frame */
1710                         /*
1711                          * Only complain about missing suitable frames if we
1712                          * haven't already found one in a previous iteration.
1713                          */
1714                         pr_err("Unable to find a suitable frame in timer @ %pa\n",
1715                                 &timer->cntctlbase);
1716         }
1717
1718         if (best_frame)
1719                 ret = arch_timer_mem_frame_register(best_frame);
1720 out:
1721         kfree(timers);
1722         return ret;
1723 }
1724
1725 /* Initialize per-processor generic timer and memory-mapped timer(if present) */
1726 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1727 {
1728         int ret, platform_timer_count;
1729
1730         if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1731                 pr_warn("already initialized, skipping\n");
1732                 return -EINVAL;
1733         }
1734
1735         arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1736
1737         ret = acpi_gtdt_init(table, &platform_timer_count);
1738         if (ret)
1739                 return ret;
1740
1741         arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI] =
1742                 acpi_gtdt_map_ppi(ARCH_TIMER_PHYS_NONSECURE_PPI);
1743
1744         arch_timer_ppi[ARCH_TIMER_VIRT_PPI] =
1745                 acpi_gtdt_map_ppi(ARCH_TIMER_VIRT_PPI);
1746
1747         arch_timer_ppi[ARCH_TIMER_HYP_PPI] =
1748                 acpi_gtdt_map_ppi(ARCH_TIMER_HYP_PPI);
1749
1750         arch_timer_populate_kvm_info();
1751
1752         /*
1753          * When probing via ACPI, we have no mechanism to override the sysreg
1754          * CNTFRQ value. This *must* be correct.
1755          */
1756         arch_timer_rate = arch_timer_get_cntfrq();
1757         ret = validate_timer_rate();
1758         if (ret) {
1759                 pr_err(FW_BUG "frequency not available.\n");
1760                 return ret;
1761         }
1762
1763         arch_timer_uses_ppi = arch_timer_select_ppi();
1764         if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1765                 pr_err("No interrupt available, giving up\n");
1766                 return -EINVAL;
1767         }
1768
1769         /* Always-on capability */
1770         arch_timer_c3stop = acpi_gtdt_c3stop(arch_timer_uses_ppi);
1771
1772         /* Check for globally applicable workarounds */
1773         arch_timer_check_ool_workaround(ate_match_acpi_oem_info, table);
1774
1775         ret = arch_timer_register();
1776         if (ret)
1777                 return ret;
1778
1779         if (platform_timer_count &&
1780             arch_timer_mem_acpi_init(platform_timer_count))
1781                 pr_err("Failed to initialize memory-mapped timer.\n");
1782
1783         return arch_timer_common_init();
1784 }
1785 TIMER_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1786 #endif
1787
1788 int kvm_arch_ptp_get_crosststamp(u64 *cycle, struct timespec64 *ts,
1789                                  struct clocksource **cs)
1790 {
1791         struct arm_smccc_res hvc_res;
1792         u32 ptp_counter;
1793         ktime_t ktime;
1794
1795         if (!IS_ENABLED(CONFIG_HAVE_ARM_SMCCC_DISCOVERY))
1796                 return -EOPNOTSUPP;
1797
1798         if (arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI)
1799                 ptp_counter = KVM_PTP_VIRT_COUNTER;
1800         else
1801                 ptp_counter = KVM_PTP_PHYS_COUNTER;
1802
1803         arm_smccc_1_1_invoke(ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID,
1804                              ptp_counter, &hvc_res);
1805
1806         if ((int)(hvc_res.a0) < 0)
1807                 return -EOPNOTSUPP;
1808
1809         ktime = (u64)hvc_res.a0 << 32 | hvc_res.a1;
1810         *ts = ktime_to_timespec64(ktime);
1811         if (cycle)
1812                 *cycle = (u64)hvc_res.a2 << 32 | hvc_res.a3;
1813         if (cs)
1814                 *cs = &clocksource_counter;
1815
1816         return 0;
1817 }
1818 EXPORT_SYMBOL_GPL(kvm_arch_ptp_get_crosststamp);