arch/arm64/kvm/psci.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2012 - ARM Ltd
   4  * Author: Marc Zyngier <marc.zyngier@arm.com>
   5  */
   6
   7 #include <linux/arm-smccc.h>
   8 #include <linux/preempt.h>
   9 #include <linux/kvm_host.h>
  10 #include <linux/uaccess.h>
  11 #include <linux/wait.h>
  12
  13 #include <asm/cputype.h>
  14 #include <asm/kvm_emulate.h>
  15
  16 #include <kvm/arm_psci.h>
  17 #include <kvm/arm_hypercalls.h>
  18
  19 /*
  20  * This is an implementation of the Power State Coordination Interface
  21  * as described in ARM document number ARM DEN 0022A.
  22  */
  23
  24 #define AFFINITY_MASK(level)    ~((0x1UL << ((level) * MPIDR_LEVEL_BITS)) - 1)
  25
  26 static unsigned long psci_affinity_mask(unsigned long affinity_level)
  27 {
  28         if (affinity_level <= 3)
  29                 return MPIDR_HWID_BITMASK & AFFINITY_MASK(affinity_level);
  30
  31         return 0;
  32 }
  33
  34 static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu)
  35 {
  36         /*
  37          * NOTE: For simplicity, we make VCPU suspend emulation to be
  38          * same-as WFI (Wait-for-interrupt) emulation.
  39          *
  40          * This means for KVM the wakeup events are interrupts and
  41          * this is consistent with intended use of StateID as described
  42          * in section 5.4.1 of PSCI v0.2 specification (ARM DEN 0022A).
  43          *
  44          * Further, we also treat power-down request to be same as
  45          * stand-by request as-per section 5.4.2 clause 3 of PSCI v0.2
  46          * specification (ARM DEN 0022A). This means all suspend states
  47          * for KVM will preserve the register state.
  48          */
  49         kvm_vcpu_wfi(vcpu);
  50
  51         return PSCI_RET_SUCCESS;
  52 }
  53
  54 static inline bool kvm_psci_valid_affinity(struct kvm_vcpu *vcpu,
  55                                            unsigned long affinity)
  56 {
  57         return !(affinity & ~MPIDR_HWID_BITMASK);
  58 }
  59
  60 static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
  61 {
  62         struct vcpu_reset_state *reset_state;
  63         struct kvm *kvm = source_vcpu->kvm;
  64         struct kvm_vcpu *vcpu = NULL;
  65         int ret = PSCI_RET_SUCCESS;
  66         unsigned long cpu_id;
  67
  68         cpu_id = smccc_get_arg1(source_vcpu);
  69         if (!kvm_psci_valid_affinity(source_vcpu, cpu_id))
  70                 return PSCI_RET_INVALID_PARAMS;
  71
  72         vcpu = kvm_mpidr_to_vcpu(kvm, cpu_id);
  73
  74         /*
  75          * Make sure the caller requested a valid CPU and that the CPU is
  76          * turned off.
  77          */
  78         if (!vcpu)
  79                 return PSCI_RET_INVALID_PARAMS;
  80
  81         spin_lock(&vcpu->arch.mp_state_lock);
  82         if (!kvm_arm_vcpu_stopped(vcpu)) {
  83                 if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1)
  84                         ret = PSCI_RET_ALREADY_ON;
  85                 else
  86                         ret = PSCI_RET_INVALID_PARAMS;
  87
  88                 goto out_unlock;
  89         }
  90
  91         reset_state = &vcpu->arch.reset_state;
  92
  93         reset_state->pc = smccc_get_arg2(source_vcpu);
  94
  95         /* Propagate caller endianness */
  96         reset_state->be = kvm_vcpu_is_be(source_vcpu);
  97
  98         /*
  99          * NOTE: We always update r0 (or x0) because for PSCI v0.1
 100          * the general purpose registers are undefined upon CPU_ON.
 101          */
 102         reset_state->r0 = smccc_get_arg3(source_vcpu);
 103
 104         reset_state->reset = true;
 105         kvm_make_request(KVM_REQ_VCPU_RESET, vcpu);
 106
 107         /*
 108          * Make sure the reset request is observed if the RUNNABLE mp_state is
 109          * observed.
 110          */
 111         smp_wmb();
 112
 113         WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
 114         kvm_vcpu_wake_up(vcpu);
 115
 116 out_unlock:
 117         spin_unlock(&vcpu->arch.mp_state_lock);
 118         return ret;
 119 }
 120
 121 static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu)
 122 {
 123         int matching_cpus = 0;
 124         unsigned long i, mpidr;
 125         unsigned long target_affinity;
 126         unsigned long target_affinity_mask;
 127         unsigned long lowest_affinity_level;
 128         struct kvm *kvm = vcpu->kvm;
 129         struct kvm_vcpu *tmp;
 130
 131         target_affinity = smccc_get_arg1(vcpu);
 132         lowest_affinity_level = smccc_get_arg2(vcpu);
 133
 134         if (!kvm_psci_valid_affinity(vcpu, target_affinity))
 135                 return PSCI_RET_INVALID_PARAMS;
 136
 137         /* Determine target affinity mask */
 138         target_affinity_mask = psci_affinity_mask(lowest_affinity_level);
 139         if (!target_affinity_mask)
 140                 return PSCI_RET_INVALID_PARAMS;
 141
 142         /* Ignore other bits of target affinity */
 143         target_affinity &= target_affinity_mask;
 144
 145         /*
 146          * If one or more VCPU matching target affinity are running
 147          * then ON else OFF
 148          */
 149         kvm_for_each_vcpu(i, tmp, kvm) {
 150                 mpidr = kvm_vcpu_get_mpidr_aff(tmp);
 151                 if ((mpidr & target_affinity_mask) == target_affinity) {
 152                         matching_cpus++;
 153                         if (!kvm_arm_vcpu_stopped(tmp))
 154                                 return PSCI_0_2_AFFINITY_LEVEL_ON;
 155                 }
 156         }
 157
 158         if (!matching_cpus)
 159                 return PSCI_RET_INVALID_PARAMS;
 160
 161         return PSCI_0_2_AFFINITY_LEVEL_OFF;
 162 }
 163
 164 static void kvm_prepare_system_event(struct kvm_vcpu *vcpu, u32 type, u64 flags)
 165 {
 166         unsigned long i;
 167         struct kvm_vcpu *tmp;
 168
 169         /*
 170          * The KVM ABI specifies that a system event exit may call KVM_RUN
 171          * again and may perform shutdown/reboot at a later time that when the
 172          * actual request is made.  Since we are implementing PSCI and a
 173          * caller of PSCI reboot and shutdown expects that the system shuts
 174          * down or reboots immediately, let's make sure that VCPUs are not run
 175          * after this call is handled and before the VCPUs have been
 176          * re-initialized.
 177          */
 178         kvm_for_each_vcpu(i, tmp, vcpu->kvm) {
 179                 spin_lock(&tmp->arch.mp_state_lock);
 180                 WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
 181                 spin_unlock(&tmp->arch.mp_state_lock);
 182         }
 183         kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP);
 184
 185         memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event));
 186         vcpu->run->system_event.type = type;
 187         vcpu->run->system_event.ndata = 1;
 188         vcpu->run->system_event.data[0] = flags;
 189         vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
 190 }
 191
 192 static void kvm_psci_system_off(struct kvm_vcpu *vcpu)
 193 {
 194         kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_SHUTDOWN, 0);
 195 }
 196
 197 static void kvm_psci_system_reset(struct kvm_vcpu *vcpu)
 198 {
 199         kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET, 0);
 200 }
 201
 202 static void kvm_psci_system_reset2(struct kvm_vcpu *vcpu)
 203 {
 204         kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET,
 205                                  KVM_SYSTEM_EVENT_RESET_FLAG_PSCI_RESET2);
 206 }
 207
 208 static void kvm_psci_system_suspend(struct kvm_vcpu *vcpu)
 209 {
 210         struct kvm_run *run = vcpu->run;
 211
 212         memset(&run->system_event, 0, sizeof(vcpu->run->system_event));
 213         run->system_event.type = KVM_SYSTEM_EVENT_SUSPEND;
 214         run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
 215 }
 216
 217 static void kvm_psci_narrow_to_32bit(struct kvm_vcpu *vcpu)
 218 {
 219         int i;
 220
 221         /*
 222          * Zero the input registers' upper 32 bits. They will be fully
 223          * zeroed on exit, so we're fine changing them in place.
 224          */
 225         for (i = 1; i < 4; i++)
 226                 vcpu_set_reg(vcpu, i, lower_32_bits(vcpu_get_reg(vcpu, i)));
 227 }
 228
 229 static unsigned long kvm_psci_check_allowed_function(struct kvm_vcpu *vcpu, u32 fn)
 230 {
 231         /*
 232          * Prevent 32 bit guests from calling 64 bit PSCI functions.
 233          */
 234         if ((fn & PSCI_0_2_64BIT) && vcpu_mode_is_32bit(vcpu))
 235                 return PSCI_RET_NOT_SUPPORTED;
 236
 237         return 0;
 238 }
 239
 240 static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu)
 241 {
 242         u32 psci_fn = smccc_get_function(vcpu);
 243         unsigned long val;
 244         int ret = 1;
 245
 246         switch (psci_fn) {
 247         case PSCI_0_2_FN_PSCI_VERSION:
 248                 /*
 249                  * Bits[31:16] = Major Version = 0
 250                  * Bits[15:0] = Minor Version = 2
 251                  */
 252                 val = KVM_ARM_PSCI_0_2;
 253                 break;
 254         case PSCI_0_2_FN_CPU_SUSPEND:
 255         case PSCI_0_2_FN64_CPU_SUSPEND:
 256                 val = kvm_psci_vcpu_suspend(vcpu);
 257                 break;
 258         case PSCI_0_2_FN_CPU_OFF:
 259                 kvm_arm_vcpu_power_off(vcpu);
 260                 val = PSCI_RET_SUCCESS;
 261                 break;
 262         case PSCI_0_2_FN_CPU_ON:
 263                 kvm_psci_narrow_to_32bit(vcpu);
 264                 fallthrough;
 265         case PSCI_0_2_FN64_CPU_ON:
 266                 val = kvm_psci_vcpu_on(vcpu);
 267                 break;
 268         case PSCI_0_2_FN_AFFINITY_INFO:
 269                 kvm_psci_narrow_to_32bit(vcpu);
 270                 fallthrough;
 271         case PSCI_0_2_FN64_AFFINITY_INFO:
 272                 val = kvm_psci_vcpu_affinity_info(vcpu);
 273                 break;
 274         case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
 275                 /*
 276                  * Trusted OS is MP hence does not require migration
 277                  * or
 278                  * Trusted OS is not present
 279                  */
 280                 val = PSCI_0_2_TOS_MP;
 281                 break;
 282         case PSCI_0_2_FN_SYSTEM_OFF:
 283                 kvm_psci_system_off(vcpu);
 284                 /*
 285                  * We shouldn't be going back to guest VCPU after
 286                  * receiving SYSTEM_OFF request.
 287                  *
 288                  * If user space accidentally/deliberately resumes
 289                  * guest VCPU after SYSTEM_OFF request then guest
 290                  * VCPU should see internal failure from PSCI return
 291                  * value. To achieve this, we preload r0 (or x0) with
 292                  * PSCI return value INTERNAL_FAILURE.
 293                  */
 294                 val = PSCI_RET_INTERNAL_FAILURE;
 295                 ret = 0;
 296                 break;
 297         case PSCI_0_2_FN_SYSTEM_RESET:
 298                 kvm_psci_system_reset(vcpu);
 299                 /*
 300                  * Same reason as SYSTEM_OFF for preloading r0 (or x0)
 301                  * with PSCI return value INTERNAL_FAILURE.
 302                  */
 303                 val = PSCI_RET_INTERNAL_FAILURE;
 304                 ret = 0;
 305                 break;
 306         default:
 307                 val = PSCI_RET_NOT_SUPPORTED;
 308                 break;
 309         }
 310
 311         smccc_set_retval(vcpu, val, 0, 0, 0);
 312         return ret;
 313 }
 314
 315 static int kvm_psci_1_x_call(struct kvm_vcpu *vcpu, u32 minor)
 316 {
 317         unsigned long val = PSCI_RET_NOT_SUPPORTED;
 318         u32 psci_fn = smccc_get_function(vcpu);
 319         struct kvm *kvm = vcpu->kvm;
 320         u32 arg;
 321         int ret = 1;
 322
 323         switch(psci_fn) {
 324         case PSCI_0_2_FN_PSCI_VERSION:
 325                 val = minor == 0 ? KVM_ARM_PSCI_1_0 : KVM_ARM_PSCI_1_1;
 326                 break;
 327         case PSCI_1_0_FN_PSCI_FEATURES:
 328                 arg = smccc_get_arg1(vcpu);
 329                 val = kvm_psci_check_allowed_function(vcpu, arg);
 330                 if (val)
 331                         break;
 332
 333                 val = PSCI_RET_NOT_SUPPORTED;
 334
 335                 switch(arg) {
 336                 case PSCI_0_2_FN_PSCI_VERSION:
 337                 case PSCI_0_2_FN_CPU_SUSPEND:
 338                 case PSCI_0_2_FN64_CPU_SUSPEND:
 339                 case PSCI_0_2_FN_CPU_OFF:
 340                 case PSCI_0_2_FN_CPU_ON:
 341                 case PSCI_0_2_FN64_CPU_ON:
 342                 case PSCI_0_2_FN_AFFINITY_INFO:
 343                 case PSCI_0_2_FN64_AFFINITY_INFO:
 344                 case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
 345                 case PSCI_0_2_FN_SYSTEM_OFF:
 346                 case PSCI_0_2_FN_SYSTEM_RESET:
 347                 case PSCI_1_0_FN_PSCI_FEATURES:
 348                 case ARM_SMCCC_VERSION_FUNC_ID:
 349                         val = 0;
 350                         break;
 351                 case PSCI_1_0_FN_SYSTEM_SUSPEND:
 352                 case PSCI_1_0_FN64_SYSTEM_SUSPEND:
 353                         if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags))
 354                                 val = 0;
 355                         break;
 356                 case PSCI_1_1_FN_SYSTEM_RESET2:
 357                 case PSCI_1_1_FN64_SYSTEM_RESET2:
 358                         if (minor >= 1)
 359                                 val = 0;
 360                         break;
 361                 }
 362                 break;
 363         case PSCI_1_0_FN_SYSTEM_SUSPEND:
 364                 kvm_psci_narrow_to_32bit(vcpu);
 365                 fallthrough;
 366         case PSCI_1_0_FN64_SYSTEM_SUSPEND:
 367                 /*
 368                  * Return directly to userspace without changing the vCPU's
 369                  * registers. Userspace depends on reading the SMCCC parameters
 370                  * to implement SYSTEM_SUSPEND.
 371                  */
 372                 if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags)) {
 373                         kvm_psci_system_suspend(vcpu);
 374                         return 0;
 375                 }
 376                 break;
 377         case PSCI_1_1_FN_SYSTEM_RESET2:
 378                 kvm_psci_narrow_to_32bit(vcpu);
 379                 fallthrough;
 380         case PSCI_1_1_FN64_SYSTEM_RESET2:
 381                 if (minor >= 1) {
 382                         arg = smccc_get_arg1(vcpu);
 383
 384                         if (arg <= PSCI_1_1_RESET_TYPE_SYSTEM_WARM_RESET ||
 385                             arg >= PSCI_1_1_RESET_TYPE_VENDOR_START) {
 386                                 kvm_psci_system_reset2(vcpu);
 387                                 vcpu_set_reg(vcpu, 0, PSCI_RET_INTERNAL_FAILURE);
 388                                 return 0;
 389                         }
 390
 391                         val = PSCI_RET_INVALID_PARAMS;
 392                         break;
 393                 }
 394                 break;
 395         default:
 396                 return kvm_psci_0_2_call(vcpu);
 397         }
 398
 399         smccc_set_retval(vcpu, val, 0, 0, 0);
 400         return ret;
 401 }
 402
 403 static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu)
 404 {
 405         u32 psci_fn = smccc_get_function(vcpu);
 406         unsigned long val;
 407
 408         switch (psci_fn) {
 409         case KVM_PSCI_FN_CPU_OFF:
 410                 kvm_arm_vcpu_power_off(vcpu);
 411                 val = PSCI_RET_SUCCESS;
 412                 break;
 413         case KVM_PSCI_FN_CPU_ON:
 414                 val = kvm_psci_vcpu_on(vcpu);
 415                 break;
 416         default:
 417                 val = PSCI_RET_NOT_SUPPORTED;
 418                 break;
 419         }
 420
 421         smccc_set_retval(vcpu, val, 0, 0, 0);
 422         return 1;
 423 }
 424
 425 /**
 426  * kvm_psci_call - handle PSCI call if r0 value is in range
 427  * @vcpu: Pointer to the VCPU struct
 428  *
 429  * Handle PSCI calls from guests through traps from HVC instructions.
 430  * The calling convention is similar to SMC calls to the secure world
 431  * where the function number is placed in r0.
 432  *
 433  * This function returns: > 0 (success), 0 (success but exit to user
 434  * space), and < 0 (errors)
 435  *
 436  * Errors:
 437  * -EINVAL: Unrecognized PSCI function
 438  */
 439 int kvm_psci_call(struct kvm_vcpu *vcpu)
 440 {
 441         u32 psci_fn = smccc_get_function(vcpu);
 442         int version = kvm_psci_version(vcpu);
 443         unsigned long val;
 444
 445         val = kvm_psci_check_allowed_function(vcpu, psci_fn);
 446         if (val) {
 447                 smccc_set_retval(vcpu, val, 0, 0, 0);
 448                 return 1;
 449         }
 450
 451         switch (version) {
 452         case KVM_ARM_PSCI_1_1:
 453                 return kvm_psci_1_x_call(vcpu, 1);
 454         case KVM_ARM_PSCI_1_0:
 455                 return kvm_psci_1_x_call(vcpu, 0);
 456         case KVM_ARM_PSCI_0_2:
 457                 return kvm_psci_0_2_call(vcpu);
 458         case KVM_ARM_PSCI_0_1:
 459                 return kvm_psci_0_1_call(vcpu);
 460         default:
 461                 WARN_ONCE(1, "Unknown PSCI version %d", version);
 462                 smccc_set_retval(vcpu, SMCCC_RET_NOT_SUPPORTED, 0, 0, 0);
 463                 return 1;
 464         }
 465 }