arch/riscv/kvm/vcpu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2019 Western Digital Corporation or its affiliates.
   4  *
   5  * Authors:
   6  *     Anup Patel <anup.patel@wdc.com>
   7  */
   8
   9 #include <linux/bitops.h>
  10 #include <linux/errno.h>
  11 #include <linux/err.h>
  12 #include <linux/kdebug.h>
  13 #include <linux/module.h>
  14 #include <linux/percpu.h>
  15 #include <linux/uaccess.h>
  16 #include <linux/vmalloc.h>
  17 #include <linux/sched/signal.h>
  18 #include <linux/fs.h>
  19 #include <linux/kvm_host.h>
  20 #include <asm/csr.h>
  21 #include <asm/hwcap.h>
  22
  23 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
  24         KVM_GENERIC_VCPU_STATS(),
  25         STATS_DESC_COUNTER(VCPU, ecall_exit_stat),
  26         STATS_DESC_COUNTER(VCPU, wfi_exit_stat),
  27         STATS_DESC_COUNTER(VCPU, mmio_exit_user),
  28         STATS_DESC_COUNTER(VCPU, mmio_exit_kernel),
  29         STATS_DESC_COUNTER(VCPU, exits)
  30 };
  31
  32 const struct kvm_stats_header kvm_vcpu_stats_header = {
  33         .name_size = KVM_STATS_NAME_SIZE,
  34         .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
  35         .id_offset = sizeof(struct kvm_stats_header),
  36         .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
  37         .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
  38                        sizeof(kvm_vcpu_stats_desc),
  39 };
  40
  41 #define KVM_RISCV_ISA_ALLOWED   (riscv_isa_extension_mask(a) | \
  42                                  riscv_isa_extension_mask(c) | \
  43                                  riscv_isa_extension_mask(d) | \
  44                                  riscv_isa_extension_mask(f) | \
  45                                  riscv_isa_extension_mask(i) | \
  46                                  riscv_isa_extension_mask(m) | \
  47                                  riscv_isa_extension_mask(s) | \
  48                                  riscv_isa_extension_mask(u))
  49
  50 static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu)
  51 {
  52         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
  53         struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr;
  54         struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
  55         struct kvm_cpu_context *reset_cntx = &vcpu->arch.guest_reset_context;
  56
  57         memcpy(csr, reset_csr, sizeof(*csr));
  58
  59         memcpy(cntx, reset_cntx, sizeof(*cntx));
  60
  61         kvm_riscv_vcpu_fp_reset(vcpu);
  62
  63         kvm_riscv_vcpu_timer_reset(vcpu);
  64
  65         WRITE_ONCE(vcpu->arch.irqs_pending, 0);
  66         WRITE_ONCE(vcpu->arch.irqs_pending_mask, 0);
  67 }
  68
  69 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
  70 {
  71         return 0;
  72 }
  73
  74 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
  75 {
  76         struct kvm_cpu_context *cntx;
  77
  78         /* Mark this VCPU never ran */
  79         vcpu->arch.ran_atleast_once = false;
  80
  81         /* Setup ISA features available to VCPU */
  82         vcpu->arch.isa = riscv_isa_extension_base(NULL) & KVM_RISCV_ISA_ALLOWED;
  83
  84         /* Setup reset state of shadow SSTATUS and HSTATUS CSRs */
  85         cntx = &vcpu->arch.guest_reset_context;
  86         cntx->sstatus = SR_SPP | SR_SPIE;
  87         cntx->hstatus = 0;
  88         cntx->hstatus |= HSTATUS_VTW;
  89         cntx->hstatus |= HSTATUS_SPVP;
  90         cntx->hstatus |= HSTATUS_SPV;
  91
  92         /* Setup VCPU timer */
  93         kvm_riscv_vcpu_timer_init(vcpu);
  94
  95         /* Reset VCPU */
  96         kvm_riscv_reset_vcpu(vcpu);
  97
  98         return 0;
  99 }
 100
 101 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 102 {
 103 }
 104
 105 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 106 {
 107         /* Cleanup VCPU timer */
 108         kvm_riscv_vcpu_timer_deinit(vcpu);
 109
 110         /* Flush the pages pre-allocated for Stage2 page table mappings */
 111         kvm_riscv_stage2_flush_cache(vcpu);
 112 }
 113
 114 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 115 {
 116         return kvm_riscv_vcpu_has_interrupts(vcpu, 1UL << IRQ_VS_TIMER);
 117 }
 118
 119 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
 120 {
 121 }
 122
 123 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
 124 {
 125 }
 126
 127 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 128 {
 129         return (kvm_riscv_vcpu_has_interrupts(vcpu, -1UL) &&
 130                 !vcpu->arch.power_off && !vcpu->arch.pause);
 131 }
 132
 133 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
 134 {
 135         return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
 136 }
 137
 138 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
 139 {
 140         return (vcpu->arch.guest_context.sstatus & SR_SPP) ? true : false;
 141 }
 142
 143 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
 144 {
 145         return VM_FAULT_SIGBUS;
 146 }
 147
 148 static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu,
 149                                          const struct kvm_one_reg *reg)
 150 {
 151         unsigned long __user *uaddr =
 152                         (unsigned long __user *)(unsigned long)reg->addr;
 153         unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 154                                             KVM_REG_SIZE_MASK |
 155                                             KVM_REG_RISCV_CONFIG);
 156         unsigned long reg_val;
 157
 158         if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 159                 return -EINVAL;
 160
 161         switch (reg_num) {
 162         case KVM_REG_RISCV_CONFIG_REG(isa):
 163                 reg_val = vcpu->arch.isa;
 164                 break;
 165         default:
 166                 return -EINVAL;
 167         }
 168
 169         if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
 170                 return -EFAULT;
 171
 172         return 0;
 173 }
 174
 175 static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu,
 176                                          const struct kvm_one_reg *reg)
 177 {
 178         unsigned long __user *uaddr =
 179                         (unsigned long __user *)(unsigned long)reg->addr;
 180         unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 181                                             KVM_REG_SIZE_MASK |
 182                                             KVM_REG_RISCV_CONFIG);
 183         unsigned long reg_val;
 184
 185         if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 186                 return -EINVAL;
 187
 188         if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
 189                 return -EFAULT;
 190
 191         switch (reg_num) {
 192         case KVM_REG_RISCV_CONFIG_REG(isa):
 193                 if (!vcpu->arch.ran_atleast_once) {
 194                         vcpu->arch.isa = reg_val;
 195                         vcpu->arch.isa &= riscv_isa_extension_base(NULL);
 196                         vcpu->arch.isa &= KVM_RISCV_ISA_ALLOWED;
 197                         kvm_riscv_vcpu_fp_reset(vcpu);
 198                 } else {
 199                         return -EOPNOTSUPP;
 200                 }
 201                 break;
 202         default:
 203                 return -EINVAL;
 204         }
 205
 206         return 0;
 207 }
 208
 209 static int kvm_riscv_vcpu_get_reg_core(struct kvm_vcpu *vcpu,
 210                                        const struct kvm_one_reg *reg)
 211 {
 212         struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
 213         unsigned long __user *uaddr =
 214                         (unsigned long __user *)(unsigned long)reg->addr;
 215         unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 216                                             KVM_REG_SIZE_MASK |
 217                                             KVM_REG_RISCV_CORE);
 218         unsigned long reg_val;
 219
 220         if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 221                 return -EINVAL;
 222         if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long))
 223                 return -EINVAL;
 224
 225         if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc))
 226                 reg_val = cntx->sepc;
 227         else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num &&
 228                  reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6))
 229                 reg_val = ((unsigned long *)cntx)[reg_num];
 230         else if (reg_num == KVM_REG_RISCV_CORE_REG(mode))
 231                 reg_val = (cntx->sstatus & SR_SPP) ?
 232                                 KVM_RISCV_MODE_S : KVM_RISCV_MODE_U;
 233         else
 234                 return -EINVAL;
 235
 236         if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
 237                 return -EFAULT;
 238
 239         return 0;
 240 }
 241
 242 static int kvm_riscv_vcpu_set_reg_core(struct kvm_vcpu *vcpu,
 243                                        const struct kvm_one_reg *reg)
 244 {
 245         struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
 246         unsigned long __user *uaddr =
 247                         (unsigned long __user *)(unsigned long)reg->addr;
 248         unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 249                                             KVM_REG_SIZE_MASK |
 250                                             KVM_REG_RISCV_CORE);
 251         unsigned long reg_val;
 252
 253         if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 254                 return -EINVAL;
 255         if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long))
 256                 return -EINVAL;
 257
 258         if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
 259                 return -EFAULT;
 260
 261         if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc))
 262                 cntx->sepc = reg_val;
 263         else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num &&
 264                  reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6))
 265                 ((unsigned long *)cntx)[reg_num] = reg_val;
 266         else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) {
 267                 if (reg_val == KVM_RISCV_MODE_S)
 268                         cntx->sstatus |= SR_SPP;
 269                 else
 270                         cntx->sstatus &= ~SR_SPP;
 271         } else
 272                 return -EINVAL;
 273
 274         return 0;
 275 }
 276
 277 static int kvm_riscv_vcpu_get_reg_csr(struct kvm_vcpu *vcpu,
 278                                       const struct kvm_one_reg *reg)
 279 {
 280         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 281         unsigned long __user *uaddr =
 282                         (unsigned long __user *)(unsigned long)reg->addr;
 283         unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 284                                             KVM_REG_SIZE_MASK |
 285                                             KVM_REG_RISCV_CSR);
 286         unsigned long reg_val;
 287
 288         if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 289                 return -EINVAL;
 290         if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long))
 291                 return -EINVAL;
 292
 293         if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) {
 294                 kvm_riscv_vcpu_flush_interrupts(vcpu);
 295                 reg_val = (csr->hvip >> VSIP_TO_HVIP_SHIFT) & VSIP_VALID_MASK;
 296         } else
 297                 reg_val = ((unsigned long *)csr)[reg_num];
 298
 299         if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
 300                 return -EFAULT;
 301
 302         return 0;
 303 }
 304
 305 static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu,
 306                                       const struct kvm_one_reg *reg)
 307 {
 308         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 309         unsigned long __user *uaddr =
 310                         (unsigned long __user *)(unsigned long)reg->addr;
 311         unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 312                                             KVM_REG_SIZE_MASK |
 313                                             KVM_REG_RISCV_CSR);
 314         unsigned long reg_val;
 315
 316         if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 317                 return -EINVAL;
 318         if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long))
 319                 return -EINVAL;
 320
 321         if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
 322                 return -EFAULT;
 323
 324         if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) {
 325                 reg_val &= VSIP_VALID_MASK;
 326                 reg_val <<= VSIP_TO_HVIP_SHIFT;
 327         }
 328
 329         ((unsigned long *)csr)[reg_num] = reg_val;
 330
 331         if (reg_num == KVM_REG_RISCV_CSR_REG(sip))
 332                 WRITE_ONCE(vcpu->arch.irqs_pending_mask, 0);
 333
 334         return 0;
 335 }
 336
 337 static int kvm_riscv_vcpu_set_reg(struct kvm_vcpu *vcpu,
 338                                   const struct kvm_one_reg *reg)
 339 {
 340         if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_CONFIG)
 341                 return kvm_riscv_vcpu_set_reg_config(vcpu, reg);
 342         else if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_CORE)
 343                 return kvm_riscv_vcpu_set_reg_core(vcpu, reg);
 344         else if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_CSR)
 345                 return kvm_riscv_vcpu_set_reg_csr(vcpu, reg);
 346         else if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_TIMER)
 347                 return kvm_riscv_vcpu_set_reg_timer(vcpu, reg);
 348         else if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_FP_F)
 349                 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg,
 350                                                  KVM_REG_RISCV_FP_F);
 351         else if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_FP_D)
 352                 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg,
 353                                                  KVM_REG_RISCV_FP_D);
 354
 355         return -EINVAL;
 356 }
 357
 358 static int kvm_riscv_vcpu_get_reg(struct kvm_vcpu *vcpu,
 359                                   const struct kvm_one_reg *reg)
 360 {
 361         if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_CONFIG)
 362                 return kvm_riscv_vcpu_get_reg_config(vcpu, reg);
 363         else if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_CORE)
 364                 return kvm_riscv_vcpu_get_reg_core(vcpu, reg);
 365         else if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_CSR)
 366                 return kvm_riscv_vcpu_get_reg_csr(vcpu, reg);
 367         else if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_TIMER)
 368                 return kvm_riscv_vcpu_get_reg_timer(vcpu, reg);
 369         else if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_FP_F)
 370                 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg,
 371                                                  KVM_REG_RISCV_FP_F);
 372         else if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_FP_D)
 373                 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg,
 374                                                  KVM_REG_RISCV_FP_D);
 375
 376         return -EINVAL;
 377 }
 378
 379 long kvm_arch_vcpu_async_ioctl(struct file *filp,
 380                                unsigned int ioctl, unsigned long arg)
 381 {
 382         struct kvm_vcpu *vcpu = filp->private_data;
 383         void __user *argp = (void __user *)arg;
 384
 385         if (ioctl == KVM_INTERRUPT) {
 386                 struct kvm_interrupt irq;
 387
 388                 if (copy_from_user(&irq, argp, sizeof(irq)))
 389                         return -EFAULT;
 390
 391                 if (irq.irq == KVM_INTERRUPT_SET)
 392                         return kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT);
 393                 else
 394                         return kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT);
 395         }
 396
 397         return -ENOIOCTLCMD;
 398 }
 399
 400 long kvm_arch_vcpu_ioctl(struct file *filp,
 401                          unsigned int ioctl, unsigned long arg)
 402 {
 403         struct kvm_vcpu *vcpu = filp->private_data;
 404         void __user *argp = (void __user *)arg;
 405         long r = -EINVAL;
 406
 407         switch (ioctl) {
 408         case KVM_SET_ONE_REG:
 409         case KVM_GET_ONE_REG: {
 410                 struct kvm_one_reg reg;
 411
 412                 r = -EFAULT;
 413                 if (copy_from_user(&reg, argp, sizeof(reg)))
 414                         break;
 415
 416                 if (ioctl == KVM_SET_ONE_REG)
 417                         r = kvm_riscv_vcpu_set_reg(vcpu, &reg);
 418                 else
 419                         r = kvm_riscv_vcpu_get_reg(vcpu, &reg);
 420                 break;
 421         }
 422         default:
 423                 break;
 424         }
 425
 426         return r;
 427 }
 428
 429 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 430                                   struct kvm_sregs *sregs)
 431 {
 432         return -EINVAL;
 433 }
 434
 435 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 436                                   struct kvm_sregs *sregs)
 437 {
 438         return -EINVAL;
 439 }
 440
 441 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 442 {
 443         return -EINVAL;
 444 }
 445
 446 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 447 {
 448         return -EINVAL;
 449 }
 450
 451 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 452                                   struct kvm_translation *tr)
 453 {
 454         return -EINVAL;
 455 }
 456
 457 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 458 {
 459         return -EINVAL;
 460 }
 461
 462 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 463 {
 464         return -EINVAL;
 465 }
 466
 467 void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu)
 468 {
 469         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 470         unsigned long mask, val;
 471
 472         if (READ_ONCE(vcpu->arch.irqs_pending_mask)) {
 473                 mask = xchg_acquire(&vcpu->arch.irqs_pending_mask, 0);
 474                 val = READ_ONCE(vcpu->arch.irqs_pending) & mask;
 475
 476                 csr->hvip &= ~mask;
 477                 csr->hvip |= val;
 478         }
 479 }
 480
 481 void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu)
 482 {
 483         unsigned long hvip;
 484         struct kvm_vcpu_arch *v = &vcpu->arch;
 485         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 486
 487         /* Read current HVIP and VSIE CSRs */
 488         csr->vsie = csr_read(CSR_VSIE);
 489
 490         /* Sync-up HVIP.VSSIP bit changes does by Guest */
 491         hvip = csr_read(CSR_HVIP);
 492         if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) {
 493                 if (hvip & (1UL << IRQ_VS_SOFT)) {
 494                         if (!test_and_set_bit(IRQ_VS_SOFT,
 495                                               &v->irqs_pending_mask))
 496                                 set_bit(IRQ_VS_SOFT, &v->irqs_pending);
 497                 } else {
 498                         if (!test_and_set_bit(IRQ_VS_SOFT,
 499                                               &v->irqs_pending_mask))
 500                                 clear_bit(IRQ_VS_SOFT, &v->irqs_pending);
 501                 }
 502         }
 503 }
 504
 505 int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
 506 {
 507         if (irq != IRQ_VS_SOFT &&
 508             irq != IRQ_VS_TIMER &&
 509             irq != IRQ_VS_EXT)
 510                 return -EINVAL;
 511
 512         set_bit(irq, &vcpu->arch.irqs_pending);
 513         smp_mb__before_atomic();
 514         set_bit(irq, &vcpu->arch.irqs_pending_mask);
 515
 516         kvm_vcpu_kick(vcpu);
 517
 518         return 0;
 519 }
 520
 521 int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
 522 {
 523         if (irq != IRQ_VS_SOFT &&
 524             irq != IRQ_VS_TIMER &&
 525             irq != IRQ_VS_EXT)
 526                 return -EINVAL;
 527
 528         clear_bit(irq, &vcpu->arch.irqs_pending);
 529         smp_mb__before_atomic();
 530         set_bit(irq, &vcpu->arch.irqs_pending_mask);
 531
 532         return 0;
 533 }
 534
 535 bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, unsigned long mask)
 536 {
 537         unsigned long ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK)
 538                             << VSIP_TO_HVIP_SHIFT) & mask;
 539
 540         return (READ_ONCE(vcpu->arch.irqs_pending) & ie) ? true : false;
 541 }
 542
 543 void kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu)
 544 {
 545         vcpu->arch.power_off = true;
 546         kvm_make_request(KVM_REQ_SLEEP, vcpu);
 547         kvm_vcpu_kick(vcpu);
 548 }
 549
 550 void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu)
 551 {
 552         vcpu->arch.power_off = false;
 553         kvm_vcpu_wake_up(vcpu);
 554 }
 555
 556 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 557                                     struct kvm_mp_state *mp_state)
 558 {
 559         if (vcpu->arch.power_off)
 560                 mp_state->mp_state = KVM_MP_STATE_STOPPED;
 561         else
 562                 mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
 563
 564         return 0;
 565 }
 566
 567 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 568                                     struct kvm_mp_state *mp_state)
 569 {
 570         int ret = 0;
 571
 572         switch (mp_state->mp_state) {
 573         case KVM_MP_STATE_RUNNABLE:
 574                 vcpu->arch.power_off = false;
 575                 break;
 576         case KVM_MP_STATE_STOPPED:
 577                 kvm_riscv_vcpu_power_off(vcpu);
 578                 break;
 579         default:
 580                 ret = -EINVAL;
 581         }
 582
 583         return ret;
 584 }
 585
 586 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 587                                         struct kvm_guest_debug *dbg)
 588 {
 589         /* TODO; To be implemented later. */
 590         return -EINVAL;
 591 }
 592
 593 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 594 {
 595         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 596
 597         csr_write(CSR_VSSTATUS, csr->vsstatus);
 598         csr_write(CSR_VSIE, csr->vsie);
 599         csr_write(CSR_VSTVEC, csr->vstvec);
 600         csr_write(CSR_VSSCRATCH, csr->vsscratch);
 601         csr_write(CSR_VSEPC, csr->vsepc);
 602         csr_write(CSR_VSCAUSE, csr->vscause);
 603         csr_write(CSR_VSTVAL, csr->vstval);
 604         csr_write(CSR_HVIP, csr->hvip);
 605         csr_write(CSR_VSATP, csr->vsatp);
 606
 607         kvm_riscv_stage2_update_hgatp(vcpu);
 608
 609         kvm_riscv_vcpu_timer_restore(vcpu);
 610
 611         kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context);
 612         kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context,
 613                                         vcpu->arch.isa);
 614
 615         vcpu->cpu = cpu;
 616 }
 617
 618 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 619 {
 620         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 621
 622         vcpu->cpu = -1;
 623
 624         kvm_riscv_vcpu_guest_fp_save(&vcpu->arch.guest_context,
 625                                      vcpu->arch.isa);
 626         kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context);
 627
 628         csr_write(CSR_HGATP, 0);
 629
 630         csr->vsstatus = csr_read(CSR_VSSTATUS);
 631         csr->vsie = csr_read(CSR_VSIE);
 632         csr->vstvec = csr_read(CSR_VSTVEC);
 633         csr->vsscratch = csr_read(CSR_VSSCRATCH);
 634         csr->vsepc = csr_read(CSR_VSEPC);
 635         csr->vscause = csr_read(CSR_VSCAUSE);
 636         csr->vstval = csr_read(CSR_VSTVAL);
 637         csr->hvip = csr_read(CSR_HVIP);
 638         csr->vsatp = csr_read(CSR_VSATP);
 639 }
 640
 641 static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)
 642 {
 643         struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
 644
 645         if (kvm_request_pending(vcpu)) {
 646                 if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) {
 647                         rcuwait_wait_event(wait,
 648                                 (!vcpu->arch.power_off) && (!vcpu->arch.pause),
 649                                 TASK_INTERRUPTIBLE);
 650
 651                         if (vcpu->arch.power_off || vcpu->arch.pause) {
 652                                 /*
 653                                  * Awaken to handle a signal, request to
 654                                  * sleep again later.
 655                                  */
 656                                 kvm_make_request(KVM_REQ_SLEEP, vcpu);
 657                         }
 658                 }
 659
 660                 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
 661                         kvm_riscv_reset_vcpu(vcpu);
 662
 663                 if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu))
 664                         kvm_riscv_stage2_update_hgatp(vcpu);
 665
 666                 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
 667                         __kvm_riscv_hfence_gvma_all();
 668         }
 669 }
 670
 671 static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu)
 672 {
 673         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 674
 675         csr_write(CSR_HVIP, csr->hvip);
 676 }
 677
 678 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 679 {
 680         int ret;
 681         struct kvm_cpu_trap trap;
 682         struct kvm_run *run = vcpu->run;
 683
 684         /* Mark this VCPU ran at least once */
 685         vcpu->arch.ran_atleast_once = true;
 686
 687         vcpu->arch.srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
 688
 689         /* Process MMIO value returned from user-space */
 690         if (run->exit_reason == KVM_EXIT_MMIO) {
 691                 ret = kvm_riscv_vcpu_mmio_return(vcpu, vcpu->run);
 692                 if (ret) {
 693                         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->arch.srcu_idx);
 694                         return ret;
 695                 }
 696         }
 697
 698         /* Process SBI value returned from user-space */
 699         if (run->exit_reason == KVM_EXIT_RISCV_SBI) {
 700                 ret = kvm_riscv_vcpu_sbi_return(vcpu, vcpu->run);
 701                 if (ret) {
 702                         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->arch.srcu_idx);
 703                         return ret;
 704                 }
 705         }
 706
 707         if (run->immediate_exit) {
 708                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->arch.srcu_idx);
 709                 return -EINTR;
 710         }
 711
 712         vcpu_load(vcpu);
 713
 714         kvm_sigset_activate(vcpu);
 715
 716         ret = 1;
 717         run->exit_reason = KVM_EXIT_UNKNOWN;
 718         while (ret > 0) {
 719                 /* Check conditions before entering the guest */
 720                 cond_resched();
 721
 722                 kvm_riscv_stage2_vmid_update(vcpu);
 723
 724                 kvm_riscv_check_vcpu_requests(vcpu);
 725
 726                 preempt_disable();
 727
 728                 local_irq_disable();
 729
 730                 /*
 731                  * Exit if we have a signal pending so that we can deliver
 732                  * the signal to user space.
 733                  */
 734                 if (signal_pending(current)) {
 735                         ret = -EINTR;
 736                         run->exit_reason = KVM_EXIT_INTR;
 737                 }
 738
 739                 /*
 740                  * Ensure we set mode to IN_GUEST_MODE after we disable
 741                  * interrupts and before the final VCPU requests check.
 742                  * See the comment in kvm_vcpu_exiting_guest_mode() and
 743                  * Documentation/virt/kvm/vcpu-requests.rst
 744                  */
 745                 vcpu->mode = IN_GUEST_MODE;
 746
 747                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->arch.srcu_idx);
 748                 smp_mb__after_srcu_read_unlock();
 749
 750                 /*
 751                  * We might have got VCPU interrupts updated asynchronously
 752                  * so update it in HW.
 753                  */
 754                 kvm_riscv_vcpu_flush_interrupts(vcpu);
 755
 756                 /* Update HVIP CSR for current CPU */
 757                 kvm_riscv_update_hvip(vcpu);
 758
 759                 if (ret <= 0 ||
 760                     kvm_riscv_stage2_vmid_ver_changed(&vcpu->kvm->arch.vmid) ||
 761                     kvm_request_pending(vcpu)) {
 762                         vcpu->mode = OUTSIDE_GUEST_MODE;
 763                         local_irq_enable();
 764                         preempt_enable();
 765                         vcpu->arch.srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
 766                         continue;
 767                 }
 768
 769                 guest_enter_irqoff();
 770
 771                 __kvm_riscv_switch_to(&vcpu->arch);
 772
 773                 vcpu->mode = OUTSIDE_GUEST_MODE;
 774                 vcpu->stat.exits++;
 775
 776                 /*
 777                  * Save SCAUSE, STVAL, HTVAL, and HTINST because we might
 778                  * get an interrupt between __kvm_riscv_switch_to() and
 779                  * local_irq_enable() which can potentially change CSRs.
 780                  */
 781                 trap.sepc = vcpu->arch.guest_context.sepc;
 782                 trap.scause = csr_read(CSR_SCAUSE);
 783                 trap.stval = csr_read(CSR_STVAL);
 784                 trap.htval = csr_read(CSR_HTVAL);
 785                 trap.htinst = csr_read(CSR_HTINST);
 786
 787                 /* Syncup interrupts state with HW */
 788                 kvm_riscv_vcpu_sync_interrupts(vcpu);
 789
 790                 /*
 791                  * We may have taken a host interrupt in VS/VU-mode (i.e.
 792                  * while executing the guest). This interrupt is still
 793                  * pending, as we haven't serviced it yet!
 794                  *
 795                  * We're now back in HS-mode with interrupts disabled
 796                  * so enabling the interrupts now will have the effect
 797                  * of taking the interrupt again, in HS-mode this time.
 798                  */
 799                 local_irq_enable();
 800
 801                 /*
 802                  * We do local_irq_enable() before calling guest_exit() so
 803                  * that if a timer interrupt hits while running the guest
 804                  * we account that tick as being spent in the guest. We
 805                  * enable preemption after calling guest_exit() so that if
 806                  * we get preempted we make sure ticks after that is not
 807                  * counted as guest time.
 808                  */
 809                 guest_exit();
 810
 811                 preempt_enable();
 812
 813                 vcpu->arch.srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
 814
 815                 ret = kvm_riscv_vcpu_exit(vcpu, run, &trap);
 816         }
 817
 818         kvm_sigset_deactivate(vcpu);
 819
 820         vcpu_put(vcpu);
 821
 822         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->arch.srcu_idx);
 823
 824         return ret;
 825 }