arch/arm64/kvm/arch_timer.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2012 ARM Ltd.
   4  * Author: Marc Zyngier <marc.zyngier@arm.com>
   5  */
   6
   7 #include <linux/cpu.h>
   8 #include <linux/kvm.h>
   9 #include <linux/kvm_host.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/irq.h>
  12 #include <linux/irqdomain.h>
  13 #include <linux/uaccess.h>
  14
  15 #include <clocksource/arm_arch_timer.h>
  16 #include <asm/arch_timer.h>
  17 #include <asm/kvm_emulate.h>
  18 #include <asm/kvm_hyp.h>
  19
  20 #include <kvm/arm_vgic.h>
  21 #include <kvm/arm_arch_timer.h>
  22
  23 #include "trace.h"
  24
  25 static struct timecounter *timecounter;
  26 static unsigned int host_vtimer_irq;
  27 static unsigned int host_ptimer_irq;
  28 static u32 host_vtimer_irq_flags;
  29 static u32 host_ptimer_irq_flags;
  30
  31 static DEFINE_STATIC_KEY_FALSE(has_gic_active_state);
  32
  33 static const struct kvm_irq_level default_ptimer_irq = {
  34         .irq    = 30,
  35         .level  = 1,
  36 };
  37
  38 static const struct kvm_irq_level default_vtimer_irq = {
  39         .irq    = 27,
  40         .level  = 1,
  41 };
  42
  43 static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx);
  44 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
  45                                  struct arch_timer_context *timer_ctx);
  46 static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx);
  47 static void kvm_arm_timer_write(struct kvm_vcpu *vcpu,
  48                                 struct arch_timer_context *timer,
  49                                 enum kvm_arch_timer_regs treg,
  50                                 u64 val);
  51 static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu,
  52                               struct arch_timer_context *timer,
  53                               enum kvm_arch_timer_regs treg);
  54
  55 u32 timer_get_ctl(struct arch_timer_context *ctxt)
  56 {
  57         struct kvm_vcpu *vcpu = ctxt->vcpu;
  58
  59         switch(arch_timer_ctx_index(ctxt)) {
  60         case TIMER_VTIMER:
  61                 return __vcpu_sys_reg(vcpu, CNTV_CTL_EL0);
  62         case TIMER_PTIMER:
  63                 return __vcpu_sys_reg(vcpu, CNTP_CTL_EL0);
  64         default:
  65                 WARN_ON(1);
  66                 return 0;
  67         }
  68 }
  69
  70 u64 timer_get_cval(struct arch_timer_context *ctxt)
  71 {
  72         struct kvm_vcpu *vcpu = ctxt->vcpu;
  73
  74         switch(arch_timer_ctx_index(ctxt)) {
  75         case TIMER_VTIMER:
  76                 return __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0);
  77         case TIMER_PTIMER:
  78                 return __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0);
  79         default:
  80                 WARN_ON(1);
  81                 return 0;
  82         }
  83 }
  84
  85 static u64 timer_get_offset(struct arch_timer_context *ctxt)
  86 {
  87         struct kvm_vcpu *vcpu = ctxt->vcpu;
  88
  89         switch(arch_timer_ctx_index(ctxt)) {
  90         case TIMER_VTIMER:
  91                 return __vcpu_sys_reg(vcpu, CNTVOFF_EL2);
  92         default:
  93                 return 0;
  94         }
  95 }
  96
  97 static void timer_set_ctl(struct arch_timer_context *ctxt, u32 ctl)
  98 {
  99         struct kvm_vcpu *vcpu = ctxt->vcpu;
 100
 101         switch(arch_timer_ctx_index(ctxt)) {
 102         case TIMER_VTIMER:
 103                 __vcpu_sys_reg(vcpu, CNTV_CTL_EL0) = ctl;
 104                 break;
 105         case TIMER_PTIMER:
 106                 __vcpu_sys_reg(vcpu, CNTP_CTL_EL0) = ctl;
 107                 break;
 108         default:
 109                 WARN_ON(1);
 110         }
 111 }
 112
 113 static void timer_set_cval(struct arch_timer_context *ctxt, u64 cval)
 114 {
 115         struct kvm_vcpu *vcpu = ctxt->vcpu;
 116
 117         switch(arch_timer_ctx_index(ctxt)) {
 118         case TIMER_VTIMER:
 119                 __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0) = cval;
 120                 break;
 121         case TIMER_PTIMER:
 122                 __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0) = cval;
 123                 break;
 124         default:
 125                 WARN_ON(1);
 126         }
 127 }
 128
 129 static void timer_set_offset(struct arch_timer_context *ctxt, u64 offset)
 130 {
 131         struct kvm_vcpu *vcpu = ctxt->vcpu;
 132
 133         switch(arch_timer_ctx_index(ctxt)) {
 134         case TIMER_VTIMER:
 135                 __vcpu_sys_reg(vcpu, CNTVOFF_EL2) = offset;
 136                 break;
 137         default:
 138                 WARN(offset, "timer %ld\n", arch_timer_ctx_index(ctxt));
 139         }
 140 }
 141
 142 u64 kvm_phys_timer_read(void)
 143 {
 144         return timecounter->cc->read(timecounter->cc);
 145 }
 146
 147 static void get_timer_map(struct kvm_vcpu *vcpu, struct timer_map *map)
 148 {
 149         if (has_vhe()) {
 150                 map->direct_vtimer = vcpu_vtimer(vcpu);
 151                 map->direct_ptimer = vcpu_ptimer(vcpu);
 152                 map->emul_ptimer = NULL;
 153         } else {
 154                 map->direct_vtimer = vcpu_vtimer(vcpu);
 155                 map->direct_ptimer = NULL;
 156                 map->emul_ptimer = vcpu_ptimer(vcpu);
 157         }
 158
 159         trace_kvm_get_timer_map(vcpu->vcpu_id, map);
 160 }
 161
 162 static inline bool userspace_irqchip(struct kvm *kvm)
 163 {
 164         return static_branch_unlikely(&userspace_irqchip_in_use) &&
 165                 unlikely(!irqchip_in_kernel(kvm));
 166 }
 167
 168 static void soft_timer_start(struct hrtimer *hrt, u64 ns)
 169 {
 170         hrtimer_start(hrt, ktime_add_ns(ktime_get(), ns),
 171                       HRTIMER_MODE_ABS_HARD);
 172 }
 173
 174 static void soft_timer_cancel(struct hrtimer *hrt)
 175 {
 176         hrtimer_cancel(hrt);
 177 }
 178
 179 static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
 180 {
 181         struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
 182         struct arch_timer_context *ctx;
 183         struct timer_map map;
 184
 185         /*
 186          * We may see a timer interrupt after vcpu_put() has been called which
 187          * sets the CPU's vcpu pointer to NULL, because even though the timer
 188          * has been disabled in timer_save_state(), the hardware interrupt
 189          * signal may not have been retired from the interrupt controller yet.
 190          */
 191         if (!vcpu)
 192                 return IRQ_HANDLED;
 193
 194         get_timer_map(vcpu, &map);
 195
 196         if (irq == host_vtimer_irq)
 197                 ctx = map.direct_vtimer;
 198         else
 199                 ctx = map.direct_ptimer;
 200
 201         if (kvm_timer_should_fire(ctx))
 202                 kvm_timer_update_irq(vcpu, true, ctx);
 203
 204         if (userspace_irqchip(vcpu->kvm) &&
 205             !static_branch_unlikely(&has_gic_active_state))
 206                 disable_percpu_irq(host_vtimer_irq);
 207
 208         return IRQ_HANDLED;
 209 }
 210
 211 static u64 kvm_counter_compute_delta(struct arch_timer_context *timer_ctx,
 212                                      u64 val)
 213 {
 214         u64 now = kvm_phys_timer_read() - timer_get_offset(timer_ctx);
 215
 216         if (now < val) {
 217                 u64 ns;
 218
 219                 ns = cyclecounter_cyc2ns(timecounter->cc,
 220                                          val - now,
 221                                          timecounter->mask,
 222                                          &timecounter->frac);
 223                 return ns;
 224         }
 225
 226         return 0;
 227 }
 228
 229 static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx)
 230 {
 231         return kvm_counter_compute_delta(timer_ctx, timer_get_cval(timer_ctx));
 232 }
 233
 234 static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx)
 235 {
 236         WARN_ON(timer_ctx && timer_ctx->loaded);
 237         return timer_ctx &&
 238                 ((timer_get_ctl(timer_ctx) &
 239                   (ARCH_TIMER_CTRL_IT_MASK | ARCH_TIMER_CTRL_ENABLE)) == ARCH_TIMER_CTRL_ENABLE);
 240 }
 241
 242 static bool vcpu_has_wfit_active(struct kvm_vcpu *vcpu)
 243 {
 244         return (cpus_have_final_cap(ARM64_HAS_WFXT) &&
 245                 (vcpu->arch.flags & KVM_ARM64_WFIT));
 246 }
 247
 248 static u64 wfit_delay_ns(struct kvm_vcpu *vcpu)
 249 {
 250         struct arch_timer_context *ctx = vcpu_vtimer(vcpu);
 251         u64 val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu));
 252
 253         return kvm_counter_compute_delta(ctx, val);
 254 }
 255
 256 /*
 257  * Returns the earliest expiration time in ns among guest timers.
 258  * Note that it will return 0 if none of timers can fire.
 259  */
 260 static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu)
 261 {
 262         u64 min_delta = ULLONG_MAX;
 263         int i;
 264
 265         for (i = 0; i < NR_KVM_TIMERS; i++) {
 266                 struct arch_timer_context *ctx = &vcpu->arch.timer_cpu.timers[i];
 267
 268                 WARN(ctx->loaded, "timer %d loaded\n", i);
 269                 if (kvm_timer_irq_can_fire(ctx))
 270                         min_delta = min(min_delta, kvm_timer_compute_delta(ctx));
 271         }
 272
 273         if (vcpu_has_wfit_active(vcpu))
 274                 min_delta = min(min_delta, wfit_delay_ns(vcpu));
 275
 276         /* If none of timers can fire, then return 0 */
 277         if (min_delta == ULLONG_MAX)
 278                 return 0;
 279
 280         return min_delta;
 281 }
 282
 283 static enum hrtimer_restart kvm_bg_timer_expire(struct hrtimer *hrt)
 284 {
 285         struct arch_timer_cpu *timer;
 286         struct kvm_vcpu *vcpu;
 287         u64 ns;
 288
 289         timer = container_of(hrt, struct arch_timer_cpu, bg_timer);
 290         vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
 291
 292         /*
 293          * Check that the timer has really expired from the guest's
 294          * PoV (NTP on the host may have forced it to expire
 295          * early). If we should have slept longer, restart it.
 296          */
 297         ns = kvm_timer_earliest_exp(vcpu);
 298         if (unlikely(ns)) {
 299                 hrtimer_forward_now(hrt, ns_to_ktime(ns));
 300                 return HRTIMER_RESTART;
 301         }
 302
 303         kvm_vcpu_wake_up(vcpu);
 304         return HRTIMER_NORESTART;
 305 }
 306
 307 static enum hrtimer_restart kvm_hrtimer_expire(struct hrtimer *hrt)
 308 {
 309         struct arch_timer_context *ctx;
 310         struct kvm_vcpu *vcpu;
 311         u64 ns;
 312
 313         ctx = container_of(hrt, struct arch_timer_context, hrtimer);
 314         vcpu = ctx->vcpu;
 315
 316         trace_kvm_timer_hrtimer_expire(ctx);
 317
 318         /*
 319          * Check that the timer has really expired from the guest's
 320          * PoV (NTP on the host may have forced it to expire
 321          * early). If not ready, schedule for a later time.
 322          */
 323         ns = kvm_timer_compute_delta(ctx);
 324         if (unlikely(ns)) {
 325                 hrtimer_forward_now(hrt, ns_to_ktime(ns));
 326                 return HRTIMER_RESTART;
 327         }
 328
 329         kvm_timer_update_irq(vcpu, true, ctx);
 330         return HRTIMER_NORESTART;
 331 }
 332
 333 static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx)
 334 {
 335         enum kvm_arch_timers index;
 336         u64 cval, now;
 337
 338         if (!timer_ctx)
 339                 return false;
 340
 341         index = arch_timer_ctx_index(timer_ctx);
 342
 343         if (timer_ctx->loaded) {
 344                 u32 cnt_ctl = 0;
 345
 346                 switch (index) {
 347                 case TIMER_VTIMER:
 348                         cnt_ctl = read_sysreg_el0(SYS_CNTV_CTL);
 349                         break;
 350                 case TIMER_PTIMER:
 351                         cnt_ctl = read_sysreg_el0(SYS_CNTP_CTL);
 352                         break;
 353                 case NR_KVM_TIMERS:
 354                         /* GCC is braindead */
 355                         cnt_ctl = 0;
 356                         break;
 357                 }
 358
 359                 return  (cnt_ctl & ARCH_TIMER_CTRL_ENABLE) &&
 360                         (cnt_ctl & ARCH_TIMER_CTRL_IT_STAT) &&
 361                        !(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK);
 362         }
 363
 364         if (!kvm_timer_irq_can_fire(timer_ctx))
 365                 return false;
 366
 367         cval = timer_get_cval(timer_ctx);
 368         now = kvm_phys_timer_read() - timer_get_offset(timer_ctx);
 369
 370         return cval <= now;
 371 }
 372
 373 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 374 {
 375         return vcpu_has_wfit_active(vcpu) && wfit_delay_ns(vcpu) == 0;
 376 }
 377
 378 /*
 379  * Reflect the timer output level into the kvm_run structure
 380  */
 381 void kvm_timer_update_run(struct kvm_vcpu *vcpu)
 382 {
 383         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 384         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 385         struct kvm_sync_regs *regs = &vcpu->run->s.regs;
 386
 387         /* Populate the device bitmap with the timer states */
 388         regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER |
 389                                     KVM_ARM_DEV_EL1_PTIMER);
 390         if (kvm_timer_should_fire(vtimer))
 391                 regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER;
 392         if (kvm_timer_should_fire(ptimer))
 393                 regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER;
 394 }
 395
 396 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
 397                                  struct arch_timer_context *timer_ctx)
 398 {
 399         int ret;
 400
 401         timer_ctx->irq.level = new_level;
 402         trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_ctx->irq.irq,
 403                                    timer_ctx->irq.level);
 404
 405         if (!userspace_irqchip(vcpu->kvm)) {
 406                 ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
 407                                           timer_ctx->irq.irq,
 408                                           timer_ctx->irq.level,
 409                                           timer_ctx);
 410                 WARN_ON(ret);
 411         }
 412 }
 413
 414 /* Only called for a fully emulated timer */
 415 static void timer_emulate(struct arch_timer_context *ctx)
 416 {
 417         bool should_fire = kvm_timer_should_fire(ctx);
 418
 419         trace_kvm_timer_emulate(ctx, should_fire);
 420
 421         if (should_fire != ctx->irq.level) {
 422                 kvm_timer_update_irq(ctx->vcpu, should_fire, ctx);
 423                 return;
 424         }
 425
 426         /*
 427          * If the timer can fire now, we don't need to have a soft timer
 428          * scheduled for the future.  If the timer cannot fire at all,
 429          * then we also don't need a soft timer.
 430          */
 431         if (!kvm_timer_irq_can_fire(ctx)) {
 432                 soft_timer_cancel(&ctx->hrtimer);
 433                 return;
 434         }
 435
 436         soft_timer_start(&ctx->hrtimer, kvm_timer_compute_delta(ctx));
 437 }
 438
 439 static void timer_save_state(struct arch_timer_context *ctx)
 440 {
 441         struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu);
 442         enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
 443         unsigned long flags;
 444
 445         if (!timer->enabled)
 446                 return;
 447
 448         local_irq_save(flags);
 449
 450         if (!ctx->loaded)
 451                 goto out;
 452
 453         switch (index) {
 454         case TIMER_VTIMER:
 455                 timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTV_CTL));
 456                 timer_set_cval(ctx, read_sysreg_el0(SYS_CNTV_CVAL));
 457
 458                 /* Disable the timer */
 459                 write_sysreg_el0(0, SYS_CNTV_CTL);
 460                 isb();
 461
 462                 break;
 463         case TIMER_PTIMER:
 464                 timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTP_CTL));
 465                 timer_set_cval(ctx, read_sysreg_el0(SYS_CNTP_CVAL));
 466
 467                 /* Disable the timer */
 468                 write_sysreg_el0(0, SYS_CNTP_CTL);
 469                 isb();
 470
 471                 break;
 472         case NR_KVM_TIMERS:
 473                 BUG();
 474         }
 475
 476         trace_kvm_timer_save_state(ctx);
 477
 478         ctx->loaded = false;
 479 out:
 480         local_irq_restore(flags);
 481 }
 482
 483 /*
 484  * Schedule the background timer before calling kvm_vcpu_halt, so that this
 485  * thread is removed from its waitqueue and made runnable when there's a timer
 486  * interrupt to handle.
 487  */
 488 static void kvm_timer_blocking(struct kvm_vcpu *vcpu)
 489 {
 490         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 491         struct timer_map map;
 492
 493         get_timer_map(vcpu, &map);
 494
 495         /*
 496          * If no timers are capable of raising interrupts (disabled or
 497          * masked), then there's no more work for us to do.
 498          */
 499         if (!kvm_timer_irq_can_fire(map.direct_vtimer) &&
 500             !kvm_timer_irq_can_fire(map.direct_ptimer) &&
 501             !kvm_timer_irq_can_fire(map.emul_ptimer) &&
 502             !vcpu_has_wfit_active(vcpu))
 503                 return;
 504
 505         /*
 506          * At least one guest time will expire. Schedule a background timer.
 507          * Set the earliest expiration time among the guest timers.
 508          */
 509         soft_timer_start(&timer->bg_timer, kvm_timer_earliest_exp(vcpu));
 510 }
 511
 512 static void kvm_timer_unblocking(struct kvm_vcpu *vcpu)
 513 {
 514         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 515
 516         soft_timer_cancel(&timer->bg_timer);
 517 }
 518
 519 static void timer_restore_state(struct arch_timer_context *ctx)
 520 {
 521         struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu);
 522         enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
 523         unsigned long flags;
 524
 525         if (!timer->enabled)
 526                 return;
 527
 528         local_irq_save(flags);
 529
 530         if (ctx->loaded)
 531                 goto out;
 532
 533         switch (index) {
 534         case TIMER_VTIMER:
 535                 write_sysreg_el0(timer_get_cval(ctx), SYS_CNTV_CVAL);
 536                 isb();
 537                 write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTV_CTL);
 538                 break;
 539         case TIMER_PTIMER:
 540                 write_sysreg_el0(timer_get_cval(ctx), SYS_CNTP_CVAL);
 541                 isb();
 542                 write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTP_CTL);
 543                 break;
 544         case NR_KVM_TIMERS:
 545                 BUG();
 546         }
 547
 548         trace_kvm_timer_restore_state(ctx);
 549
 550         ctx->loaded = true;
 551 out:
 552         local_irq_restore(flags);
 553 }
 554
 555 static void set_cntvoff(u64 cntvoff)
 556 {
 557         kvm_call_hyp(__kvm_timer_set_cntvoff, cntvoff);
 558 }
 559
 560 static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, bool active)
 561 {
 562         int r;
 563         r = irq_set_irqchip_state(ctx->host_timer_irq, IRQCHIP_STATE_ACTIVE, active);
 564         WARN_ON(r);
 565 }
 566
 567 static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)
 568 {
 569         struct kvm_vcpu *vcpu = ctx->vcpu;
 570         bool phys_active = false;
 571
 572         /*
 573          * Update the timer output so that it is likely to match the
 574          * state we're about to restore. If the timer expires between
 575          * this point and the register restoration, we'll take the
 576          * interrupt anyway.
 577          */
 578         kvm_timer_update_irq(ctx->vcpu, kvm_timer_should_fire(ctx), ctx);
 579
 580         if (irqchip_in_kernel(vcpu->kvm))
 581                 phys_active = kvm_vgic_map_is_active(vcpu, ctx->irq.irq);
 582
 583         phys_active |= ctx->irq.level;
 584
 585         set_timer_irq_phys_active(ctx, phys_active);
 586 }
 587
 588 static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu)
 589 {
 590         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 591
 592         /*
 593          * Update the timer output so that it is likely to match the
 594          * state we're about to restore. If the timer expires between
 595          * this point and the register restoration, we'll take the
 596          * interrupt anyway.
 597          */
 598         kvm_timer_update_irq(vcpu, kvm_timer_should_fire(vtimer), vtimer);
 599
 600         /*
 601          * When using a userspace irqchip with the architected timers and a
 602          * host interrupt controller that doesn't support an active state, we
 603          * must still prevent continuously exiting from the guest, and
 604          * therefore mask the physical interrupt by disabling it on the host
 605          * interrupt controller when the virtual level is high, such that the
 606          * guest can make forward progress.  Once we detect the output level
 607          * being de-asserted, we unmask the interrupt again so that we exit
 608          * from the guest when the timer fires.
 609          */
 610         if (vtimer->irq.level)
 611                 disable_percpu_irq(host_vtimer_irq);
 612         else
 613                 enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
 614 }
 615
 616 void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu)
 617 {
 618         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 619         struct timer_map map;
 620
 621         if (unlikely(!timer->enabled))
 622                 return;
 623
 624         get_timer_map(vcpu, &map);
 625
 626         if (static_branch_likely(&has_gic_active_state)) {
 627                 kvm_timer_vcpu_load_gic(map.direct_vtimer);
 628                 if (map.direct_ptimer)
 629                         kvm_timer_vcpu_load_gic(map.direct_ptimer);
 630         } else {
 631                 kvm_timer_vcpu_load_nogic(vcpu);
 632         }
 633
 634         set_cntvoff(timer_get_offset(map.direct_vtimer));
 635
 636         kvm_timer_unblocking(vcpu);
 637
 638         timer_restore_state(map.direct_vtimer);
 639         if (map.direct_ptimer)
 640                 timer_restore_state(map.direct_ptimer);
 641
 642         if (map.emul_ptimer)
 643                 timer_emulate(map.emul_ptimer);
 644 }
 645
 646 bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
 647 {
 648         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 649         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 650         struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
 651         bool vlevel, plevel;
 652
 653         if (likely(irqchip_in_kernel(vcpu->kvm)))
 654                 return false;
 655
 656         vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER;
 657         plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER;
 658
 659         return kvm_timer_should_fire(vtimer) != vlevel ||
 660                kvm_timer_should_fire(ptimer) != plevel;
 661 }
 662
 663 void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
 664 {
 665         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 666         struct timer_map map;
 667
 668         if (unlikely(!timer->enabled))
 669                 return;
 670
 671         get_timer_map(vcpu, &map);
 672
 673         timer_save_state(map.direct_vtimer);
 674         if (map.direct_ptimer)
 675                 timer_save_state(map.direct_ptimer);
 676
 677         /*
 678          * Cancel soft timer emulation, because the only case where we
 679          * need it after a vcpu_put is in the context of a sleeping VCPU, and
 680          * in that case we already factor in the deadline for the physical
 681          * timer when scheduling the bg_timer.
 682          *
 683          * In any case, we re-schedule the hrtimer for the physical timer when
 684          * coming back to the VCPU thread in kvm_timer_vcpu_load().
 685          */
 686         if (map.emul_ptimer)
 687                 soft_timer_cancel(&map.emul_ptimer->hrtimer);
 688
 689         if (kvm_vcpu_is_blocking(vcpu))
 690                 kvm_timer_blocking(vcpu);
 691
 692         /*
 693          * The kernel may decide to run userspace after calling vcpu_put, so
 694          * we reset cntvoff to 0 to ensure a consistent read between user
 695          * accesses to the virtual counter and kernel access to the physical
 696          * counter of non-VHE case. For VHE, the virtual counter uses a fixed
 697          * virtual offset of zero, so no need to zero CNTVOFF_EL2 register.
 698          */
 699         set_cntvoff(0);
 700 }
 701
 702 /*
 703  * With a userspace irqchip we have to check if the guest de-asserted the
 704  * timer and if so, unmask the timer irq signal on the host interrupt
 705  * controller to ensure that we see future timer signals.
 706  */
 707 static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu)
 708 {
 709         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 710
 711         if (!kvm_timer_should_fire(vtimer)) {
 712                 kvm_timer_update_irq(vcpu, false, vtimer);
 713                 if (static_branch_likely(&has_gic_active_state))
 714                         set_timer_irq_phys_active(vtimer, false);
 715                 else
 716                         enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
 717         }
 718 }
 719
 720 void kvm_timer_sync_user(struct kvm_vcpu *vcpu)
 721 {
 722         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 723
 724         if (unlikely(!timer->enabled))
 725                 return;
 726
 727         if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
 728                 unmask_vtimer_irq_user(vcpu);
 729 }
 730
 731 int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu)
 732 {
 733         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 734         struct timer_map map;
 735
 736         get_timer_map(vcpu, &map);
 737
 738         /*
 739          * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8
 740          * and to 0 for ARMv7.  We provide an implementation that always
 741          * resets the timer to be disabled and unmasked and is compliant with
 742          * the ARMv7 architecture.
 743          */
 744         timer_set_ctl(vcpu_vtimer(vcpu), 0);
 745         timer_set_ctl(vcpu_ptimer(vcpu), 0);
 746
 747         if (timer->enabled) {
 748                 kvm_timer_update_irq(vcpu, false, vcpu_vtimer(vcpu));
 749                 kvm_timer_update_irq(vcpu, false, vcpu_ptimer(vcpu));
 750
 751                 if (irqchip_in_kernel(vcpu->kvm)) {
 752                         kvm_vgic_reset_mapped_irq(vcpu, map.direct_vtimer->irq.irq);
 753                         if (map.direct_ptimer)
 754                                 kvm_vgic_reset_mapped_irq(vcpu, map.direct_ptimer->irq.irq);
 755                 }
 756         }
 757
 758         if (map.emul_ptimer)
 759                 soft_timer_cancel(&map.emul_ptimer->hrtimer);
 760
 761         return 0;
 762 }
 763
 764 /* Make the updates of cntvoff for all vtimer contexts atomic */
 765 static void update_vtimer_cntvoff(struct kvm_vcpu *vcpu, u64 cntvoff)
 766 {
 767         unsigned long i;
 768         struct kvm *kvm = vcpu->kvm;
 769         struct kvm_vcpu *tmp;
 770
 771         mutex_lock(&kvm->lock);
 772         kvm_for_each_vcpu(i, tmp, kvm)
 773                 timer_set_offset(vcpu_vtimer(tmp), cntvoff);
 774
 775         /*
 776          * When called from the vcpu create path, the CPU being created is not
 777          * included in the loop above, so we just set it here as well.
 778          */
 779         timer_set_offset(vcpu_vtimer(vcpu), cntvoff);
 780         mutex_unlock(&kvm->lock);
 781 }
 782
 783 void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
 784 {
 785         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 786         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 787         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 788
 789         vtimer->vcpu = vcpu;
 790         ptimer->vcpu = vcpu;
 791
 792         /* Synchronize cntvoff across all vtimers of a VM. */
 793         update_vtimer_cntvoff(vcpu, kvm_phys_timer_read());
 794         timer_set_offset(ptimer, 0);
 795
 796         hrtimer_init(&timer->bg_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
 797         timer->bg_timer.function = kvm_bg_timer_expire;
 798
 799         hrtimer_init(&vtimer->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
 800         hrtimer_init(&ptimer->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
 801         vtimer->hrtimer.function = kvm_hrtimer_expire;
 802         ptimer->hrtimer.function = kvm_hrtimer_expire;
 803
 804         vtimer->irq.irq = default_vtimer_irq.irq;
 805         ptimer->irq.irq = default_ptimer_irq.irq;
 806
 807         vtimer->host_timer_irq = host_vtimer_irq;
 808         ptimer->host_timer_irq = host_ptimer_irq;
 809
 810         vtimer->host_timer_irq_flags = host_vtimer_irq_flags;
 811         ptimer->host_timer_irq_flags = host_ptimer_irq_flags;
 812 }
 813
 814 static void kvm_timer_init_interrupt(void *info)
 815 {
 816         enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
 817         enable_percpu_irq(host_ptimer_irq, host_ptimer_irq_flags);
 818 }
 819
 820 int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
 821 {
 822         struct arch_timer_context *timer;
 823
 824         switch (regid) {
 825         case KVM_REG_ARM_TIMER_CTL:
 826                 timer = vcpu_vtimer(vcpu);
 827                 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value);
 828                 break;
 829         case KVM_REG_ARM_TIMER_CNT:
 830                 timer = vcpu_vtimer(vcpu);
 831                 update_vtimer_cntvoff(vcpu, kvm_phys_timer_read() - value);
 832                 break;
 833         case KVM_REG_ARM_TIMER_CVAL:
 834                 timer = vcpu_vtimer(vcpu);
 835                 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value);
 836                 break;
 837         case KVM_REG_ARM_PTIMER_CTL:
 838                 timer = vcpu_ptimer(vcpu);
 839                 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value);
 840                 break;
 841         case KVM_REG_ARM_PTIMER_CVAL:
 842                 timer = vcpu_ptimer(vcpu);
 843                 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value);
 844                 break;
 845
 846         default:
 847                 return -1;
 848         }
 849
 850         return 0;
 851 }
 852
 853 static u64 read_timer_ctl(struct arch_timer_context *timer)
 854 {
 855         /*
 856          * Set ISTATUS bit if it's expired.
 857          * Note that according to ARMv8 ARM Issue A.k, ISTATUS bit is
 858          * UNKNOWN when ENABLE bit is 0, so we chose to set ISTATUS bit
 859          * regardless of ENABLE bit for our implementation convenience.
 860          */
 861         u32 ctl = timer_get_ctl(timer);
 862
 863         if (!kvm_timer_compute_delta(timer))
 864                 ctl |= ARCH_TIMER_CTRL_IT_STAT;
 865
 866         return ctl;
 867 }
 868
 869 u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
 870 {
 871         switch (regid) {
 872         case KVM_REG_ARM_TIMER_CTL:
 873                 return kvm_arm_timer_read(vcpu,
 874                                           vcpu_vtimer(vcpu), TIMER_REG_CTL);
 875         case KVM_REG_ARM_TIMER_CNT:
 876                 return kvm_arm_timer_read(vcpu,
 877                                           vcpu_vtimer(vcpu), TIMER_REG_CNT);
 878         case KVM_REG_ARM_TIMER_CVAL:
 879                 return kvm_arm_timer_read(vcpu,
 880                                           vcpu_vtimer(vcpu), TIMER_REG_CVAL);
 881         case KVM_REG_ARM_PTIMER_CTL:
 882                 return kvm_arm_timer_read(vcpu,
 883                                           vcpu_ptimer(vcpu), TIMER_REG_CTL);
 884         case KVM_REG_ARM_PTIMER_CNT:
 885                 return kvm_arm_timer_read(vcpu,
 886                                           vcpu_ptimer(vcpu), TIMER_REG_CNT);
 887         case KVM_REG_ARM_PTIMER_CVAL:
 888                 return kvm_arm_timer_read(vcpu,
 889                                           vcpu_ptimer(vcpu), TIMER_REG_CVAL);
 890         }
 891         return (u64)-1;
 892 }
 893
 894 static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu,
 895                               struct arch_timer_context *timer,
 896                               enum kvm_arch_timer_regs treg)
 897 {
 898         u64 val;
 899
 900         switch (treg) {
 901         case TIMER_REG_TVAL:
 902                 val = timer_get_cval(timer) - kvm_phys_timer_read() + timer_get_offset(timer);
 903                 val = lower_32_bits(val);
 904                 break;
 905
 906         case TIMER_REG_CTL:
 907                 val = read_timer_ctl(timer);
 908                 break;
 909
 910         case TIMER_REG_CVAL:
 911                 val = timer_get_cval(timer);
 912                 break;
 913
 914         case TIMER_REG_CNT:
 915                 val = kvm_phys_timer_read() - timer_get_offset(timer);
 916                 break;
 917
 918         default:
 919                 BUG();
 920         }
 921
 922         return val;
 923 }
 924
 925 u64 kvm_arm_timer_read_sysreg(struct kvm_vcpu *vcpu,
 926                               enum kvm_arch_timers tmr,
 927                               enum kvm_arch_timer_regs treg)
 928 {
 929         u64 val;
 930
 931         preempt_disable();
 932         kvm_timer_vcpu_put(vcpu);
 933
 934         val = kvm_arm_timer_read(vcpu, vcpu_get_timer(vcpu, tmr), treg);
 935
 936         kvm_timer_vcpu_load(vcpu);
 937         preempt_enable();
 938
 939         return val;
 940 }
 941
 942 static void kvm_arm_timer_write(struct kvm_vcpu *vcpu,
 943                                 struct arch_timer_context *timer,
 944                                 enum kvm_arch_timer_regs treg,
 945                                 u64 val)
 946 {
 947         switch (treg) {
 948         case TIMER_REG_TVAL:
 949                 timer_set_cval(timer, kvm_phys_timer_read() - timer_get_offset(timer) + (s32)val);
 950                 break;
 951
 952         case TIMER_REG_CTL:
 953                 timer_set_ctl(timer, val & ~ARCH_TIMER_CTRL_IT_STAT);
 954                 break;
 955
 956         case TIMER_REG_CVAL:
 957                 timer_set_cval(timer, val);
 958                 break;
 959
 960         default:
 961                 BUG();
 962         }
 963 }
 964
 965 void kvm_arm_timer_write_sysreg(struct kvm_vcpu *vcpu,
 966                                 enum kvm_arch_timers tmr,
 967                                 enum kvm_arch_timer_regs treg,
 968                                 u64 val)
 969 {
 970         preempt_disable();
 971         kvm_timer_vcpu_put(vcpu);
 972
 973         kvm_arm_timer_write(vcpu, vcpu_get_timer(vcpu, tmr), treg, val);
 974
 975         kvm_timer_vcpu_load(vcpu);
 976         preempt_enable();
 977 }
 978
 979 static int kvm_timer_starting_cpu(unsigned int cpu)
 980 {
 981         kvm_timer_init_interrupt(NULL);
 982         return 0;
 983 }
 984
 985 static int kvm_timer_dying_cpu(unsigned int cpu)
 986 {
 987         disable_percpu_irq(host_vtimer_irq);
 988         return 0;
 989 }
 990
 991 static int timer_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
 992 {
 993         if (vcpu)
 994                 irqd_set_forwarded_to_vcpu(d);
 995         else
 996                 irqd_clr_forwarded_to_vcpu(d);
 997
 998         return 0;
 999 }
1000
1001 static int timer_irq_set_irqchip_state(struct irq_data *d,
1002                                        enum irqchip_irq_state which, bool val)
1003 {
1004         if (which != IRQCHIP_STATE_ACTIVE || !irqd_is_forwarded_to_vcpu(d))
1005                 return irq_chip_set_parent_state(d, which, val);
1006
1007         if (val)
1008                 irq_chip_mask_parent(d);
1009         else
1010                 irq_chip_unmask_parent(d);
1011
1012         return 0;
1013 }
1014
1015 static void timer_irq_eoi(struct irq_data *d)
1016 {
1017         if (!irqd_is_forwarded_to_vcpu(d))
1018                 irq_chip_eoi_parent(d);
1019 }
1020
1021 static void timer_irq_ack(struct irq_data *d)
1022 {
1023         d = d->parent_data;
1024         if (d->chip->irq_ack)
1025                 d->chip->irq_ack(d);
1026 }
1027
1028 static struct irq_chip timer_chip = {
1029         .name                   = "KVM",
1030         .irq_ack                = timer_irq_ack,
1031         .irq_mask               = irq_chip_mask_parent,
1032         .irq_unmask             = irq_chip_unmask_parent,
1033         .irq_eoi                = timer_irq_eoi,
1034         .irq_set_type           = irq_chip_set_type_parent,
1035         .irq_set_vcpu_affinity  = timer_irq_set_vcpu_affinity,
1036         .irq_set_irqchip_state  = timer_irq_set_irqchip_state,
1037 };
1038
1039 static int timer_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
1040                                   unsigned int nr_irqs, void *arg)
1041 {
1042         irq_hw_number_t hwirq = (uintptr_t)arg;
1043
1044         return irq_domain_set_hwirq_and_chip(domain, virq, hwirq,
1045                                              &timer_chip, NULL);
1046 }
1047
1048 static void timer_irq_domain_free(struct irq_domain *domain, unsigned int virq,
1049                                   unsigned int nr_irqs)
1050 {
1051 }
1052
1053 static const struct irq_domain_ops timer_domain_ops = {
1054         .alloc  = timer_irq_domain_alloc,
1055         .free   = timer_irq_domain_free,
1056 };
1057
1058 static struct irq_ops arch_timer_irq_ops = {
1059         .get_input_level = kvm_arch_timer_get_input_level,
1060 };
1061
1062 static void kvm_irq_fixup_flags(unsigned int virq, u32 *flags)
1063 {
1064         *flags = irq_get_trigger_type(virq);
1065         if (*flags != IRQF_TRIGGER_HIGH && *flags != IRQF_TRIGGER_LOW) {
1066                 kvm_err("Invalid trigger for timer IRQ%d, assuming level low\n",
1067                         virq);
1068                 *flags = IRQF_TRIGGER_LOW;
1069         }
1070 }
1071
1072 static int kvm_irq_init(struct arch_timer_kvm_info *info)
1073 {
1074         struct irq_domain *domain = NULL;
1075
1076         if (info->virtual_irq <= 0) {
1077                 kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n",
1078                         info->virtual_irq);
1079                 return -ENODEV;
1080         }
1081
1082         host_vtimer_irq = info->virtual_irq;
1083         kvm_irq_fixup_flags(host_vtimer_irq, &host_vtimer_irq_flags);
1084
1085         if (kvm_vgic_global_state.no_hw_deactivation) {
1086                 struct fwnode_handle *fwnode;
1087                 struct irq_data *data;
1088
1089                 fwnode = irq_domain_alloc_named_fwnode("kvm-timer");
1090                 if (!fwnode)
1091                         return -ENOMEM;
1092
1093                 /* Assume both vtimer and ptimer in the same parent */
1094                 data = irq_get_irq_data(host_vtimer_irq);
1095                 domain = irq_domain_create_hierarchy(data->domain, 0,
1096                                                      NR_KVM_TIMERS, fwnode,
1097                                                      &timer_domain_ops, NULL);
1098                 if (!domain) {
1099                         irq_domain_free_fwnode(fwnode);
1100                         return -ENOMEM;
1101                 }
1102
1103                 arch_timer_irq_ops.flags |= VGIC_IRQ_SW_RESAMPLE;
1104                 WARN_ON(irq_domain_push_irq(domain, host_vtimer_irq,
1105                                             (void *)TIMER_VTIMER));
1106         }
1107
1108         if (info->physical_irq > 0) {
1109                 host_ptimer_irq = info->physical_irq;
1110                 kvm_irq_fixup_flags(host_ptimer_irq, &host_ptimer_irq_flags);
1111
1112                 if (domain)
1113                         WARN_ON(irq_domain_push_irq(domain, host_ptimer_irq,
1114                                                     (void *)TIMER_PTIMER));
1115         }
1116
1117         return 0;
1118 }
1119
1120 int kvm_timer_hyp_init(bool has_gic)
1121 {
1122         struct arch_timer_kvm_info *info;
1123         int err;
1124
1125         info = arch_timer_get_kvm_info();
1126         timecounter = &info->timecounter;
1127
1128         if (!timecounter->cc) {
1129                 kvm_err("kvm_arch_timer: uninitialized timecounter\n");
1130                 return -ENODEV;
1131         }
1132
1133         err = kvm_irq_init(info);
1134         if (err)
1135                 return err;
1136
1137         /* First, do the virtual EL1 timer irq */
1138
1139         err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler,
1140                                  "kvm guest vtimer", kvm_get_running_vcpus());
1141         if (err) {
1142                 kvm_err("kvm_arch_timer: can't request vtimer interrupt %d (%d)\n",
1143                         host_vtimer_irq, err);
1144                 return err;
1145         }
1146
1147         if (has_gic) {
1148                 err = irq_set_vcpu_affinity(host_vtimer_irq,
1149                                             kvm_get_running_vcpus());
1150                 if (err) {
1151                         kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
1152                         goto out_free_irq;
1153                 }
1154
1155                 static_branch_enable(&has_gic_active_state);
1156         }
1157
1158         kvm_debug("virtual timer IRQ%d\n", host_vtimer_irq);
1159
1160         /* Now let's do the physical EL1 timer irq */
1161
1162         if (info->physical_irq > 0) {
1163                 err = request_percpu_irq(host_ptimer_irq, kvm_arch_timer_handler,
1164                                          "kvm guest ptimer", kvm_get_running_vcpus());
1165                 if (err) {
1166                         kvm_err("kvm_arch_timer: can't request ptimer interrupt %d (%d)\n",
1167                                 host_ptimer_irq, err);
1168                         return err;
1169                 }
1170
1171                 if (has_gic) {
1172                         err = irq_set_vcpu_affinity(host_ptimer_irq,
1173                                                     kvm_get_running_vcpus());
1174                         if (err) {
1175                                 kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
1176                                 goto out_free_irq;
1177                         }
1178                 }
1179
1180                 kvm_debug("physical timer IRQ%d\n", host_ptimer_irq);
1181         } else if (has_vhe()) {
1182                 kvm_err("kvm_arch_timer: invalid physical timer IRQ: %d\n",
1183                         info->physical_irq);
1184                 err = -ENODEV;
1185                 goto out_free_irq;
1186         }
1187
1188         cpuhp_setup_state(CPUHP_AP_KVM_ARM_TIMER_STARTING,
1189                           "kvm/arm/timer:starting", kvm_timer_starting_cpu,
1190                           kvm_timer_dying_cpu);
1191         return 0;
1192 out_free_irq:
1193         free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus());
1194         return err;
1195 }
1196
1197 void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
1198 {
1199         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
1200
1201         soft_timer_cancel(&timer->bg_timer);
1202 }
1203
1204 static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu)
1205 {
1206         int vtimer_irq, ptimer_irq, ret;
1207         unsigned long i;
1208
1209         vtimer_irq = vcpu_vtimer(vcpu)->irq.irq;
1210         ret = kvm_vgic_set_owner(vcpu, vtimer_irq, vcpu_vtimer(vcpu));
1211         if (ret)
1212                 return false;
1213
1214         ptimer_irq = vcpu_ptimer(vcpu)->irq.irq;
1215         ret = kvm_vgic_set_owner(vcpu, ptimer_irq, vcpu_ptimer(vcpu));
1216         if (ret)
1217                 return false;
1218
1219         kvm_for_each_vcpu(i, vcpu, vcpu->kvm) {
1220                 if (vcpu_vtimer(vcpu)->irq.irq != vtimer_irq ||
1221                     vcpu_ptimer(vcpu)->irq.irq != ptimer_irq)
1222                         return false;
1223         }
1224
1225         return true;
1226 }
1227
1228 bool kvm_arch_timer_get_input_level(int vintid)
1229 {
1230         struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
1231         struct arch_timer_context *timer;
1232
1233         if (vintid == vcpu_vtimer(vcpu)->irq.irq)
1234                 timer = vcpu_vtimer(vcpu);
1235         else if (vintid == vcpu_ptimer(vcpu)->irq.irq)
1236                 timer = vcpu_ptimer(vcpu);
1237         else
1238                 BUG();
1239
1240         return kvm_timer_should_fire(timer);
1241 }
1242
1243 int kvm_timer_enable(struct kvm_vcpu *vcpu)
1244 {
1245         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
1246         struct timer_map map;
1247         int ret;
1248
1249         if (timer->enabled)
1250                 return 0;
1251
1252         /* Without a VGIC we do not map virtual IRQs to physical IRQs */
1253         if (!irqchip_in_kernel(vcpu->kvm))
1254                 goto no_vgic;
1255
1256         /*
1257          * At this stage, we have the guarantee that the vgic is both
1258          * available and initialized.
1259          */
1260         if (!timer_irqs_are_valid(vcpu)) {
1261                 kvm_debug("incorrectly configured timer irqs\n");
1262                 return -EINVAL;
1263         }
1264
1265         get_timer_map(vcpu, &map);
1266
1267         ret = kvm_vgic_map_phys_irq(vcpu,
1268                                     map.direct_vtimer->host_timer_irq,
1269                                     map.direct_vtimer->irq.irq,
1270                                     &arch_timer_irq_ops);
1271         if (ret)
1272                 return ret;
1273
1274         if (map.direct_ptimer) {
1275                 ret = kvm_vgic_map_phys_irq(vcpu,
1276                                             map.direct_ptimer->host_timer_irq,
1277                                             map.direct_ptimer->irq.irq,
1278                                             &arch_timer_irq_ops);
1279         }
1280
1281         if (ret)
1282                 return ret;
1283
1284 no_vgic:
1285         timer->enabled = 1;
1286         return 0;
1287 }
1288
1289 /*
1290  * On VHE system, we only need to configure the EL2 timer trap register once,
1291  * not for every world switch.
1292  * The host kernel runs at EL2 with HCR_EL2.TGE == 1,
1293  * and this makes those bits have no effect for the host kernel execution.
1294  */
1295 void kvm_timer_init_vhe(void)
1296 {
1297         /* When HCR_EL2.E2H ==1, EL1PCEN and EL1PCTEN are shifted by 10 */
1298         u32 cnthctl_shift = 10;
1299         u64 val;
1300
1301         /*
1302          * VHE systems allow the guest direct access to the EL1 physical
1303          * timer/counter.
1304          */
1305         val = read_sysreg(cnthctl_el2);
1306         val |= (CNTHCTL_EL1PCEN << cnthctl_shift);
1307         val |= (CNTHCTL_EL1PCTEN << cnthctl_shift);
1308         write_sysreg(val, cnthctl_el2);
1309 }
1310
1311 static void set_timer_irqs(struct kvm *kvm, int vtimer_irq, int ptimer_irq)
1312 {
1313         struct kvm_vcpu *vcpu;
1314         unsigned long i;
1315
1316         kvm_for_each_vcpu(i, vcpu, kvm) {
1317                 vcpu_vtimer(vcpu)->irq.irq = vtimer_irq;
1318                 vcpu_ptimer(vcpu)->irq.irq = ptimer_irq;
1319         }
1320 }
1321
1322 int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1323 {
1324         int __user *uaddr = (int __user *)(long)attr->addr;
1325         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
1326         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
1327         int irq;
1328
1329         if (!irqchip_in_kernel(vcpu->kvm))
1330                 return -EINVAL;
1331
1332         if (get_user(irq, uaddr))
1333                 return -EFAULT;
1334
1335         if (!(irq_is_ppi(irq)))
1336                 return -EINVAL;
1337
1338         if (vcpu->arch.timer_cpu.enabled)
1339                 return -EBUSY;
1340
1341         switch (attr->attr) {
1342         case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
1343                 set_timer_irqs(vcpu->kvm, irq, ptimer->irq.irq);
1344                 break;
1345         case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
1346                 set_timer_irqs(vcpu->kvm, vtimer->irq.irq, irq);
1347                 break;
1348         default:
1349                 return -ENXIO;
1350         }
1351
1352         return 0;
1353 }
1354
1355 int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1356 {
1357         int __user *uaddr = (int __user *)(long)attr->addr;
1358         struct arch_timer_context *timer;
1359         int irq;
1360
1361         switch (attr->attr) {
1362         case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
1363                 timer = vcpu_vtimer(vcpu);
1364                 break;
1365         case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
1366                 timer = vcpu_ptimer(vcpu);
1367                 break;
1368         default:
1369                 return -ENXIO;
1370         }
1371
1372         irq = timer->irq.irq;
1373         return put_user(irq, uaddr);
1374 }
1375
1376 int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1377 {
1378         switch (attr->attr) {
1379         case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
1380         case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
1381                 return 0;
1382         }
1383
1384         return -ENXIO;
1385 }