1 // SPDX-License-Identifier: GPL-2.0
3 * hosting IBM Z kernel virtual machines (s390x)
5 * Copyright IBM Corp. 2008, 2020
7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com>
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
10 * Christian Ehrhardt <ehrhardt@de.ibm.com>
11 * Jason J. Herne <jjherne@us.ibm.com>
14 #define KMSG_COMPONENT "kvm-s390"
15 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
17 #include <linux/compiler.h>
18 #include <linux/err.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/mman.h>
25 #include <linux/module.h>
26 #include <linux/moduleparam.h>
27 #include <linux/random.h>
28 #include <linux/slab.h>
29 #include <linux/timer.h>
30 #include <linux/vmalloc.h>
31 #include <linux/bitmap.h>
32 #include <linux/sched/signal.h>
33 #include <linux/string.h>
34 #include <linux/pgtable.h>
36 #include <asm/asm-offsets.h>
37 #include <asm/lowcore.h>
41 #include <asm/switch_to.h>
44 #include <asm/cpacf.h>
45 #include <asm/timex.h>
48 #include <asm/fpu/api.h>
52 #define CREATE_TRACE_POINTS
54 #include "trace-s390.h"
56 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
58 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
59 (KVM_MAX_VCPUS + LOCAL_IRQS))
61 const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
62 KVM_GENERIC_VM_STATS(),
63 STATS_DESC_COUNTER(VM, inject_io),
64 STATS_DESC_COUNTER(VM, inject_float_mchk),
65 STATS_DESC_COUNTER(VM, inject_pfault_done),
66 STATS_DESC_COUNTER(VM, inject_service_signal),
67 STATS_DESC_COUNTER(VM, inject_virtio)
70 const struct kvm_stats_header kvm_vm_stats_header = {
71 .name_size = KVM_STATS_NAME_SIZE,
72 .num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
73 .id_offset = sizeof(struct kvm_stats_header),
74 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
75 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
76 sizeof(kvm_vm_stats_desc),
79 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
80 KVM_GENERIC_VCPU_STATS(),
81 STATS_DESC_COUNTER(VCPU, exit_userspace),
82 STATS_DESC_COUNTER(VCPU, exit_null),
83 STATS_DESC_COUNTER(VCPU, exit_external_request),
84 STATS_DESC_COUNTER(VCPU, exit_io_request),
85 STATS_DESC_COUNTER(VCPU, exit_external_interrupt),
86 STATS_DESC_COUNTER(VCPU, exit_stop_request),
87 STATS_DESC_COUNTER(VCPU, exit_validity),
88 STATS_DESC_COUNTER(VCPU, exit_instruction),
89 STATS_DESC_COUNTER(VCPU, exit_pei),
90 STATS_DESC_COUNTER(VCPU, halt_no_poll_steal),
91 STATS_DESC_COUNTER(VCPU, instruction_lctl),
92 STATS_DESC_COUNTER(VCPU, instruction_lctlg),
93 STATS_DESC_COUNTER(VCPU, instruction_stctl),
94 STATS_DESC_COUNTER(VCPU, instruction_stctg),
95 STATS_DESC_COUNTER(VCPU, exit_program_interruption),
96 STATS_DESC_COUNTER(VCPU, exit_instr_and_program),
97 STATS_DESC_COUNTER(VCPU, exit_operation_exception),
98 STATS_DESC_COUNTER(VCPU, deliver_ckc),
99 STATS_DESC_COUNTER(VCPU, deliver_cputm),
100 STATS_DESC_COUNTER(VCPU, deliver_external_call),
101 STATS_DESC_COUNTER(VCPU, deliver_emergency_signal),
102 STATS_DESC_COUNTER(VCPU, deliver_service_signal),
103 STATS_DESC_COUNTER(VCPU, deliver_virtio),
104 STATS_DESC_COUNTER(VCPU, deliver_stop_signal),
105 STATS_DESC_COUNTER(VCPU, deliver_prefix_signal),
106 STATS_DESC_COUNTER(VCPU, deliver_restart_signal),
107 STATS_DESC_COUNTER(VCPU, deliver_program),
108 STATS_DESC_COUNTER(VCPU, deliver_io),
109 STATS_DESC_COUNTER(VCPU, deliver_machine_check),
110 STATS_DESC_COUNTER(VCPU, exit_wait_state),
111 STATS_DESC_COUNTER(VCPU, inject_ckc),
112 STATS_DESC_COUNTER(VCPU, inject_cputm),
113 STATS_DESC_COUNTER(VCPU, inject_external_call),
114 STATS_DESC_COUNTER(VCPU, inject_emergency_signal),
115 STATS_DESC_COUNTER(VCPU, inject_mchk),
116 STATS_DESC_COUNTER(VCPU, inject_pfault_init),
117 STATS_DESC_COUNTER(VCPU, inject_program),
118 STATS_DESC_COUNTER(VCPU, inject_restart),
119 STATS_DESC_COUNTER(VCPU, inject_set_prefix),
120 STATS_DESC_COUNTER(VCPU, inject_stop_signal),
121 STATS_DESC_COUNTER(VCPU, instruction_epsw),
122 STATS_DESC_COUNTER(VCPU, instruction_gs),
123 STATS_DESC_COUNTER(VCPU, instruction_io_other),
124 STATS_DESC_COUNTER(VCPU, instruction_lpsw),
125 STATS_DESC_COUNTER(VCPU, instruction_lpswe),
126 STATS_DESC_COUNTER(VCPU, instruction_pfmf),
127 STATS_DESC_COUNTER(VCPU, instruction_ptff),
128 STATS_DESC_COUNTER(VCPU, instruction_sck),
129 STATS_DESC_COUNTER(VCPU, instruction_sckpf),
130 STATS_DESC_COUNTER(VCPU, instruction_stidp),
131 STATS_DESC_COUNTER(VCPU, instruction_spx),
132 STATS_DESC_COUNTER(VCPU, instruction_stpx),
133 STATS_DESC_COUNTER(VCPU, instruction_stap),
134 STATS_DESC_COUNTER(VCPU, instruction_iske),
135 STATS_DESC_COUNTER(VCPU, instruction_ri),
136 STATS_DESC_COUNTER(VCPU, instruction_rrbe),
137 STATS_DESC_COUNTER(VCPU, instruction_sske),
138 STATS_DESC_COUNTER(VCPU, instruction_ipte_interlock),
139 STATS_DESC_COUNTER(VCPU, instruction_stsi),
140 STATS_DESC_COUNTER(VCPU, instruction_stfl),
141 STATS_DESC_COUNTER(VCPU, instruction_tb),
142 STATS_DESC_COUNTER(VCPU, instruction_tpi),
143 STATS_DESC_COUNTER(VCPU, instruction_tprot),
144 STATS_DESC_COUNTER(VCPU, instruction_tsch),
145 STATS_DESC_COUNTER(VCPU, instruction_sie),
146 STATS_DESC_COUNTER(VCPU, instruction_essa),
147 STATS_DESC_COUNTER(VCPU, instruction_sthyi),
148 STATS_DESC_COUNTER(VCPU, instruction_sigp_sense),
149 STATS_DESC_COUNTER(VCPU, instruction_sigp_sense_running),
150 STATS_DESC_COUNTER(VCPU, instruction_sigp_external_call),
151 STATS_DESC_COUNTER(VCPU, instruction_sigp_emergency),
152 STATS_DESC_COUNTER(VCPU, instruction_sigp_cond_emergency),
153 STATS_DESC_COUNTER(VCPU, instruction_sigp_start),
154 STATS_DESC_COUNTER(VCPU, instruction_sigp_stop),
155 STATS_DESC_COUNTER(VCPU, instruction_sigp_stop_store_status),
156 STATS_DESC_COUNTER(VCPU, instruction_sigp_store_status),
157 STATS_DESC_COUNTER(VCPU, instruction_sigp_store_adtl_status),
158 STATS_DESC_COUNTER(VCPU, instruction_sigp_arch),
159 STATS_DESC_COUNTER(VCPU, instruction_sigp_prefix),
160 STATS_DESC_COUNTER(VCPU, instruction_sigp_restart),
161 STATS_DESC_COUNTER(VCPU, instruction_sigp_init_cpu_reset),
162 STATS_DESC_COUNTER(VCPU, instruction_sigp_cpu_reset),
163 STATS_DESC_COUNTER(VCPU, instruction_sigp_unknown),
164 STATS_DESC_COUNTER(VCPU, instruction_diagnose_10),
165 STATS_DESC_COUNTER(VCPU, instruction_diagnose_44),
166 STATS_DESC_COUNTER(VCPU, instruction_diagnose_9c),
167 STATS_DESC_COUNTER(VCPU, diag_9c_ignored),
168 STATS_DESC_COUNTER(VCPU, diag_9c_forward),
169 STATS_DESC_COUNTER(VCPU, instruction_diagnose_258),
170 STATS_DESC_COUNTER(VCPU, instruction_diagnose_308),
171 STATS_DESC_COUNTER(VCPU, instruction_diagnose_500),
172 STATS_DESC_COUNTER(VCPU, instruction_diagnose_other),
173 STATS_DESC_COUNTER(VCPU, pfault_sync)
176 const struct kvm_stats_header kvm_vcpu_stats_header = {
177 .name_size = KVM_STATS_NAME_SIZE,
178 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
179 .id_offset = sizeof(struct kvm_stats_header),
180 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
181 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
182 sizeof(kvm_vcpu_stats_desc),
185 /* allow nested virtualization in KVM (if enabled by user space) */
187 module_param(nested, int, S_IRUGO);
188 MODULE_PARM_DESC(nested, "Nested virtualization support");
190 /* allow 1m huge page guest backing, if !nested */
192 module_param(hpage, int, 0444);
193 MODULE_PARM_DESC(hpage, "1m huge page backing support");
195 /* maximum percentage of steal time for polling. >100 is treated like 100 */
196 static u8 halt_poll_max_steal = 10;
197 module_param(halt_poll_max_steal, byte, 0644);
198 MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
200 /* if set to true, the GISA will be initialized and used if available */
201 static bool use_gisa = true;
202 module_param(use_gisa, bool, 0644);
203 MODULE_PARM_DESC(use_gisa, "Use the GISA if the host supports it.");
205 /* maximum diag9c forwarding per second */
206 unsigned int diag9c_forwarding_hz;
207 module_param(diag9c_forwarding_hz, uint, 0644);
208 MODULE_PARM_DESC(diag9c_forwarding_hz, "Maximum diag9c forwarding per second, 0 to turn off");
211 * For now we handle at most 16 double words as this is what the s390 base
212 * kernel handles and stores in the prefix page. If we ever need to go beyond
213 * this, this requires changes to code, but the external uapi can stay.
215 #define SIZE_INTERNAL 16
218 * Base feature mask that defines default mask for facilities. Consists of the
219 * defines in FACILITIES_KVM and the non-hypervisor managed bits.
221 static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
223 * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
224 * and defines the facilities that can be enabled via a cpu model.
226 static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
228 static unsigned long kvm_s390_fac_size(void)
230 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
231 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
232 BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
233 sizeof(stfle_fac_list));
235 return SIZE_INTERNAL;
238 /* available cpu features supported by kvm */
239 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
240 /* available subfunctions indicated via query / "test bit" */
241 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
243 static struct gmap_notifier gmap_notifier;
244 static struct gmap_notifier vsie_gmap_notifier;
245 debug_info_t *kvm_s390_dbf;
246 debug_info_t *kvm_s390_dbf_uv;
248 /* Section: not file related */
249 int kvm_arch_hardware_enable(void)
251 /* every s390 is virtualization enabled ;-) */
255 int kvm_arch_check_processor_compat(void *opaque)
260 /* forward declarations */
261 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
263 static int sca_switch_to_extended(struct kvm *kvm);
265 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
270 * The TOD jumps by delta, we have to compensate this by adding
271 * -delta to the epoch.
275 /* sign-extension - we're adding to signed values below */
280 if (scb->ecd & ECD_MEF) {
281 scb->epdx += delta_idx;
282 if (scb->epoch < delta)
288 * This callback is executed during stop_machine(). All CPUs are therefore
289 * temporarily stopped. In order not to change guest behavior, we have to
290 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
291 * so a CPU won't be stopped while calculating with the epoch.
293 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
297 struct kvm_vcpu *vcpu;
299 unsigned long long *delta = v;
301 list_for_each_entry(kvm, &vm_list, vm_list) {
302 kvm_for_each_vcpu(i, vcpu, kvm) {
303 kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
305 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
306 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
308 if (vcpu->arch.cputm_enabled)
309 vcpu->arch.cputm_start += *delta;
310 if (vcpu->arch.vsie_block)
311 kvm_clock_sync_scb(vcpu->arch.vsie_block,
318 static struct notifier_block kvm_clock_notifier = {
319 .notifier_call = kvm_clock_sync,
322 int kvm_arch_hardware_setup(void *opaque)
324 gmap_notifier.notifier_call = kvm_gmap_notifier;
325 gmap_register_pte_notifier(&gmap_notifier);
326 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
327 gmap_register_pte_notifier(&vsie_gmap_notifier);
328 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
329 &kvm_clock_notifier);
333 void kvm_arch_hardware_unsetup(void)
335 gmap_unregister_pte_notifier(&gmap_notifier);
336 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
337 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
338 &kvm_clock_notifier);
341 static void allow_cpu_feat(unsigned long nr)
343 set_bit_inv(nr, kvm_s390_available_cpu_feat);
346 static inline int plo_test_bit(unsigned char nr)
348 unsigned long function = (unsigned long)nr | 0x100;
352 " lgr 0,%[function]\n"
353 /* Parameter registers are ignored for "test bit" */
358 : [function] "d" (function)
363 static __always_inline void __insn32_query(unsigned int opcode, u8 *query)
368 /* Parameter registers are ignored */
369 " .insn rrf,%[opc] << 16,2,4,6,0\n"
371 : [query] "d" ((unsigned long)query), [opc] "i" (opcode)
372 : "cc", "memory", "0", "1");
375 #define INSN_SORTL 0xb938
376 #define INSN_DFLTCC 0xb939
378 static void kvm_s390_cpu_feat_init(void)
382 for (i = 0; i < 256; ++i) {
384 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
387 if (test_facility(28)) /* TOD-clock steering */
388 ptff(kvm_s390_available_subfunc.ptff,
389 sizeof(kvm_s390_available_subfunc.ptff),
392 if (test_facility(17)) { /* MSA */
393 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
394 kvm_s390_available_subfunc.kmac);
395 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
396 kvm_s390_available_subfunc.kmc);
397 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
398 kvm_s390_available_subfunc.km);
399 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
400 kvm_s390_available_subfunc.kimd);
401 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
402 kvm_s390_available_subfunc.klmd);
404 if (test_facility(76)) /* MSA3 */
405 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
406 kvm_s390_available_subfunc.pckmo);
407 if (test_facility(77)) { /* MSA4 */
408 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
409 kvm_s390_available_subfunc.kmctr);
410 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
411 kvm_s390_available_subfunc.kmf);
412 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
413 kvm_s390_available_subfunc.kmo);
414 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
415 kvm_s390_available_subfunc.pcc);
417 if (test_facility(57)) /* MSA5 */
418 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
419 kvm_s390_available_subfunc.ppno);
421 if (test_facility(146)) /* MSA8 */
422 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
423 kvm_s390_available_subfunc.kma);
425 if (test_facility(155)) /* MSA9 */
426 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
427 kvm_s390_available_subfunc.kdsa);
429 if (test_facility(150)) /* SORTL */
430 __insn32_query(INSN_SORTL, kvm_s390_available_subfunc.sortl);
432 if (test_facility(151)) /* DFLTCC */
433 __insn32_query(INSN_DFLTCC, kvm_s390_available_subfunc.dfltcc);
435 if (MACHINE_HAS_ESOP)
436 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
438 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
439 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
441 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
442 !test_facility(3) || !nested)
444 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
445 if (sclp.has_64bscao)
446 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
448 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
450 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
452 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
454 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
456 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
458 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
460 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
462 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
463 * all skey handling functions read/set the skey from the PGSTE
464 * instead of the real storage key.
466 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
467 * pages being detected as preserved although they are resident.
469 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
470 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
472 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
473 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
474 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
476 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
477 * cannot easily shadow the SCA because of the ipte lock.
481 int kvm_arch_init(void *opaque)
485 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
489 kvm_s390_dbf_uv = debug_register("kvm-uv", 32, 1, 7 * sizeof(long));
490 if (!kvm_s390_dbf_uv)
493 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view) ||
494 debug_register_view(kvm_s390_dbf_uv, &debug_sprintf_view))
497 kvm_s390_cpu_feat_init();
499 /* Register floating interrupt controller interface. */
500 rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
502 pr_err("A FLIC registration call failed with rc=%d\n", rc);
506 rc = kvm_s390_gib_init(GAL_ISC);
517 void kvm_arch_exit(void)
519 kvm_s390_gib_destroy();
520 debug_unregister(kvm_s390_dbf);
521 debug_unregister(kvm_s390_dbf_uv);
524 /* Section: device related */
525 long kvm_arch_dev_ioctl(struct file *filp,
526 unsigned int ioctl, unsigned long arg)
528 if (ioctl == KVM_S390_ENABLE_SIE)
529 return s390_enable_sie();
533 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
538 case KVM_CAP_S390_PSW:
539 case KVM_CAP_S390_GMAP:
540 case KVM_CAP_SYNC_MMU:
541 #ifdef CONFIG_KVM_S390_UCONTROL
542 case KVM_CAP_S390_UCONTROL:
544 case KVM_CAP_ASYNC_PF:
545 case KVM_CAP_SYNC_REGS:
546 case KVM_CAP_ONE_REG:
547 case KVM_CAP_ENABLE_CAP:
548 case KVM_CAP_S390_CSS_SUPPORT:
549 case KVM_CAP_IOEVENTFD:
550 case KVM_CAP_DEVICE_CTRL:
551 case KVM_CAP_S390_IRQCHIP:
552 case KVM_CAP_VM_ATTRIBUTES:
553 case KVM_CAP_MP_STATE:
554 case KVM_CAP_IMMEDIATE_EXIT:
555 case KVM_CAP_S390_INJECT_IRQ:
556 case KVM_CAP_S390_USER_SIGP:
557 case KVM_CAP_S390_USER_STSI:
558 case KVM_CAP_S390_SKEYS:
559 case KVM_CAP_S390_IRQ_STATE:
560 case KVM_CAP_S390_USER_INSTR0:
561 case KVM_CAP_S390_CMMA_MIGRATION:
562 case KVM_CAP_S390_AIS:
563 case KVM_CAP_S390_AIS_MIGRATION:
564 case KVM_CAP_S390_VCPU_RESETS:
565 case KVM_CAP_SET_GUEST_DEBUG:
566 case KVM_CAP_S390_DIAG318:
569 case KVM_CAP_SET_GUEST_DEBUG2:
570 r = KVM_GUESTDBG_VALID_MASK;
572 case KVM_CAP_S390_HPAGE_1M:
574 if (hpage && !kvm_is_ucontrol(kvm))
577 case KVM_CAP_S390_MEM_OP:
580 case KVM_CAP_NR_VCPUS:
581 case KVM_CAP_MAX_VCPUS:
582 case KVM_CAP_MAX_VCPU_ID:
583 r = KVM_S390_BSCA_CPU_SLOTS;
584 if (!kvm_s390_use_sca_entries())
586 else if (sclp.has_esca && sclp.has_64bscao)
587 r = KVM_S390_ESCA_CPU_SLOTS;
589 case KVM_CAP_S390_COW:
590 r = MACHINE_HAS_ESOP;
592 case KVM_CAP_S390_VECTOR_REGISTERS:
595 case KVM_CAP_S390_RI:
596 r = test_facility(64);
598 case KVM_CAP_S390_GS:
599 r = test_facility(133);
601 case KVM_CAP_S390_BPB:
602 r = test_facility(82);
604 case KVM_CAP_S390_PROTECTED:
605 r = is_prot_virt_host();
613 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
616 gfn_t cur_gfn, last_gfn;
617 unsigned long gaddr, vmaddr;
618 struct gmap *gmap = kvm->arch.gmap;
619 DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
621 /* Loop over all guest segments */
622 cur_gfn = memslot->base_gfn;
623 last_gfn = memslot->base_gfn + memslot->npages;
624 for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
625 gaddr = gfn_to_gpa(cur_gfn);
626 vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
627 if (kvm_is_error_hva(vmaddr))
630 bitmap_zero(bitmap, _PAGE_ENTRIES);
631 gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
632 for (i = 0; i < _PAGE_ENTRIES; i++) {
633 if (test_bit(i, bitmap))
634 mark_page_dirty(kvm, cur_gfn + i);
637 if (fatal_signal_pending(current))
643 /* Section: vm related */
644 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
647 * Get (and clear) the dirty memory log for a memory slot.
649 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
650 struct kvm_dirty_log *log)
654 struct kvm_memory_slot *memslot;
657 if (kvm_is_ucontrol(kvm))
660 mutex_lock(&kvm->slots_lock);
663 if (log->slot >= KVM_USER_MEM_SLOTS)
666 r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot);
670 /* Clear the dirty log */
672 n = kvm_dirty_bitmap_bytes(memslot);
673 memset(memslot->dirty_bitmap, 0, n);
677 mutex_unlock(&kvm->slots_lock);
681 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
684 struct kvm_vcpu *vcpu;
686 kvm_for_each_vcpu(i, vcpu, kvm) {
687 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
691 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
699 case KVM_CAP_S390_IRQCHIP:
700 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
701 kvm->arch.use_irqchip = 1;
704 case KVM_CAP_S390_USER_SIGP:
705 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
706 kvm->arch.user_sigp = 1;
709 case KVM_CAP_S390_VECTOR_REGISTERS:
710 mutex_lock(&kvm->lock);
711 if (kvm->created_vcpus) {
713 } else if (MACHINE_HAS_VX) {
714 set_kvm_facility(kvm->arch.model.fac_mask, 129);
715 set_kvm_facility(kvm->arch.model.fac_list, 129);
716 if (test_facility(134)) {
717 set_kvm_facility(kvm->arch.model.fac_mask, 134);
718 set_kvm_facility(kvm->arch.model.fac_list, 134);
720 if (test_facility(135)) {
721 set_kvm_facility(kvm->arch.model.fac_mask, 135);
722 set_kvm_facility(kvm->arch.model.fac_list, 135);
724 if (test_facility(148)) {
725 set_kvm_facility(kvm->arch.model.fac_mask, 148);
726 set_kvm_facility(kvm->arch.model.fac_list, 148);
728 if (test_facility(152)) {
729 set_kvm_facility(kvm->arch.model.fac_mask, 152);
730 set_kvm_facility(kvm->arch.model.fac_list, 152);
732 if (test_facility(192)) {
733 set_kvm_facility(kvm->arch.model.fac_mask, 192);
734 set_kvm_facility(kvm->arch.model.fac_list, 192);
739 mutex_unlock(&kvm->lock);
740 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
741 r ? "(not available)" : "(success)");
743 case KVM_CAP_S390_RI:
745 mutex_lock(&kvm->lock);
746 if (kvm->created_vcpus) {
748 } else if (test_facility(64)) {
749 set_kvm_facility(kvm->arch.model.fac_mask, 64);
750 set_kvm_facility(kvm->arch.model.fac_list, 64);
753 mutex_unlock(&kvm->lock);
754 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
755 r ? "(not available)" : "(success)");
757 case KVM_CAP_S390_AIS:
758 mutex_lock(&kvm->lock);
759 if (kvm->created_vcpus) {
762 set_kvm_facility(kvm->arch.model.fac_mask, 72);
763 set_kvm_facility(kvm->arch.model.fac_list, 72);
766 mutex_unlock(&kvm->lock);
767 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
768 r ? "(not available)" : "(success)");
770 case KVM_CAP_S390_GS:
772 mutex_lock(&kvm->lock);
773 if (kvm->created_vcpus) {
775 } else if (test_facility(133)) {
776 set_kvm_facility(kvm->arch.model.fac_mask, 133);
777 set_kvm_facility(kvm->arch.model.fac_list, 133);
780 mutex_unlock(&kvm->lock);
781 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
782 r ? "(not available)" : "(success)");
784 case KVM_CAP_S390_HPAGE_1M:
785 mutex_lock(&kvm->lock);
786 if (kvm->created_vcpus)
788 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
792 mmap_write_lock(kvm->mm);
793 kvm->mm->context.allow_gmap_hpage_1m = 1;
794 mmap_write_unlock(kvm->mm);
796 * We might have to create fake 4k page
797 * tables. To avoid that the hardware works on
798 * stale PGSTEs, we emulate these instructions.
800 kvm->arch.use_skf = 0;
801 kvm->arch.use_pfmfi = 0;
803 mutex_unlock(&kvm->lock);
804 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
805 r ? "(not available)" : "(success)");
807 case KVM_CAP_S390_USER_STSI:
808 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
809 kvm->arch.user_stsi = 1;
812 case KVM_CAP_S390_USER_INSTR0:
813 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
814 kvm->arch.user_instr0 = 1;
815 icpt_operexc_on_all_vcpus(kvm);
825 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
829 switch (attr->attr) {
830 case KVM_S390_VM_MEM_LIMIT_SIZE:
832 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
833 kvm->arch.mem_limit);
834 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
844 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
848 switch (attr->attr) {
849 case KVM_S390_VM_MEM_ENABLE_CMMA:
854 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
855 mutex_lock(&kvm->lock);
856 if (kvm->created_vcpus)
858 else if (kvm->mm->context.allow_gmap_hpage_1m)
861 kvm->arch.use_cmma = 1;
862 /* Not compatible with cmma. */
863 kvm->arch.use_pfmfi = 0;
866 mutex_unlock(&kvm->lock);
868 case KVM_S390_VM_MEM_CLR_CMMA:
873 if (!kvm->arch.use_cmma)
876 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
877 mutex_lock(&kvm->lock);
878 idx = srcu_read_lock(&kvm->srcu);
879 s390_reset_cmma(kvm->arch.gmap->mm);
880 srcu_read_unlock(&kvm->srcu, idx);
881 mutex_unlock(&kvm->lock);
884 case KVM_S390_VM_MEM_LIMIT_SIZE: {
885 unsigned long new_limit;
887 if (kvm_is_ucontrol(kvm))
890 if (get_user(new_limit, (u64 __user *)attr->addr))
893 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
894 new_limit > kvm->arch.mem_limit)
900 /* gmap_create takes last usable address */
901 if (new_limit != KVM_S390_NO_MEM_LIMIT)
905 mutex_lock(&kvm->lock);
906 if (!kvm->created_vcpus) {
907 /* gmap_create will round the limit up */
908 struct gmap *new = gmap_create(current->mm, new_limit);
913 gmap_remove(kvm->arch.gmap);
915 kvm->arch.gmap = new;
919 mutex_unlock(&kvm->lock);
920 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
921 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
922 (void *) kvm->arch.gmap->asce);
932 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
934 void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
936 struct kvm_vcpu *vcpu;
939 kvm_s390_vcpu_block_all(kvm);
941 kvm_for_each_vcpu(i, vcpu, kvm) {
942 kvm_s390_vcpu_crypto_setup(vcpu);
943 /* recreate the shadow crycb by leaving the VSIE handler */
944 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
947 kvm_s390_vcpu_unblock_all(kvm);
950 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
952 mutex_lock(&kvm->lock);
953 switch (attr->attr) {
954 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
955 if (!test_kvm_facility(kvm, 76)) {
956 mutex_unlock(&kvm->lock);
960 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
961 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
962 kvm->arch.crypto.aes_kw = 1;
963 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
965 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
966 if (!test_kvm_facility(kvm, 76)) {
967 mutex_unlock(&kvm->lock);
971 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
972 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
973 kvm->arch.crypto.dea_kw = 1;
974 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
976 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
977 if (!test_kvm_facility(kvm, 76)) {
978 mutex_unlock(&kvm->lock);
981 kvm->arch.crypto.aes_kw = 0;
982 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
983 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
984 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
986 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
987 if (!test_kvm_facility(kvm, 76)) {
988 mutex_unlock(&kvm->lock);
991 kvm->arch.crypto.dea_kw = 0;
992 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
993 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
994 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
996 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
997 if (!ap_instructions_available()) {
998 mutex_unlock(&kvm->lock);
1001 kvm->arch.crypto.apie = 1;
1003 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1004 if (!ap_instructions_available()) {
1005 mutex_unlock(&kvm->lock);
1008 kvm->arch.crypto.apie = 0;
1011 mutex_unlock(&kvm->lock);
1015 kvm_s390_vcpu_crypto_reset_all(kvm);
1016 mutex_unlock(&kvm->lock);
1020 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
1023 struct kvm_vcpu *vcpu;
1025 kvm_for_each_vcpu(cx, vcpu, kvm)
1026 kvm_s390_sync_request(req, vcpu);
1030 * Must be called with kvm->srcu held to avoid races on memslots, and with
1031 * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
1033 static int kvm_s390_vm_start_migration(struct kvm *kvm)
1035 struct kvm_memory_slot *ms;
1036 struct kvm_memslots *slots;
1037 unsigned long ram_pages = 0;
1040 /* migration mode already enabled */
1041 if (kvm->arch.migration_mode)
1043 slots = kvm_memslots(kvm);
1044 if (!slots || !slots->used_slots)
1047 if (!kvm->arch.use_cmma) {
1048 kvm->arch.migration_mode = 1;
1051 /* mark all the pages in active slots as dirty */
1052 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
1053 ms = slots->memslots + slotnr;
1054 if (!ms->dirty_bitmap)
1057 * The second half of the bitmap is only used on x86,
1058 * and would be wasted otherwise, so we put it to good
1059 * use here to keep track of the state of the storage
1062 memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
1063 ram_pages += ms->npages;
1065 atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
1066 kvm->arch.migration_mode = 1;
1067 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
1072 * Must be called with kvm->slots_lock to avoid races with ourselves and
1073 * kvm_s390_vm_start_migration.
1075 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
1077 /* migration mode already disabled */
1078 if (!kvm->arch.migration_mode)
1080 kvm->arch.migration_mode = 0;
1081 if (kvm->arch.use_cmma)
1082 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
1086 static int kvm_s390_vm_set_migration(struct kvm *kvm,
1087 struct kvm_device_attr *attr)
1091 mutex_lock(&kvm->slots_lock);
1092 switch (attr->attr) {
1093 case KVM_S390_VM_MIGRATION_START:
1094 res = kvm_s390_vm_start_migration(kvm);
1096 case KVM_S390_VM_MIGRATION_STOP:
1097 res = kvm_s390_vm_stop_migration(kvm);
1102 mutex_unlock(&kvm->slots_lock);
1107 static int kvm_s390_vm_get_migration(struct kvm *kvm,
1108 struct kvm_device_attr *attr)
1110 u64 mig = kvm->arch.migration_mode;
1112 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1115 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1120 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1122 struct kvm_s390_vm_tod_clock gtod;
1124 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod)))
1127 if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
1129 kvm_s390_set_tod_clock(kvm, >od);
1131 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1132 gtod.epoch_idx, gtod.tod);
1137 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1141 if (copy_from_user(>od_high, (void __user *)attr->addr,
1147 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
1152 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1154 struct kvm_s390_vm_tod_clock gtod = { 0 };
1156 if (copy_from_user(>od.tod, (void __user *)attr->addr,
1160 kvm_s390_set_tod_clock(kvm, >od);
1161 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1165 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1172 switch (attr->attr) {
1173 case KVM_S390_VM_TOD_EXT:
1174 ret = kvm_s390_set_tod_ext(kvm, attr);
1176 case KVM_S390_VM_TOD_HIGH:
1177 ret = kvm_s390_set_tod_high(kvm, attr);
1179 case KVM_S390_VM_TOD_LOW:
1180 ret = kvm_s390_set_tod_low(kvm, attr);
1189 static void kvm_s390_get_tod_clock(struct kvm *kvm,
1190 struct kvm_s390_vm_tod_clock *gtod)
1192 union tod_clock clk;
1196 store_tod_clock_ext(&clk);
1198 gtod->tod = clk.tod + kvm->arch.epoch;
1199 gtod->epoch_idx = 0;
1200 if (test_kvm_facility(kvm, 139)) {
1201 gtod->epoch_idx = clk.ei + kvm->arch.epdx;
1202 if (gtod->tod < clk.tod)
1203 gtod->epoch_idx += 1;
1209 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1211 struct kvm_s390_vm_tod_clock gtod;
1213 memset(>od, 0, sizeof(gtod));
1214 kvm_s390_get_tod_clock(kvm, >od);
1215 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
1218 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1219 gtod.epoch_idx, gtod.tod);
1223 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1227 if (copy_to_user((void __user *)attr->addr, >od_high,
1230 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1235 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1239 gtod = kvm_s390_get_tod_clock_fast(kvm);
1240 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
1242 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1247 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1254 switch (attr->attr) {
1255 case KVM_S390_VM_TOD_EXT:
1256 ret = kvm_s390_get_tod_ext(kvm, attr);
1258 case KVM_S390_VM_TOD_HIGH:
1259 ret = kvm_s390_get_tod_high(kvm, attr);
1261 case KVM_S390_VM_TOD_LOW:
1262 ret = kvm_s390_get_tod_low(kvm, attr);
1271 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1273 struct kvm_s390_vm_cpu_processor *proc;
1274 u16 lowest_ibc, unblocked_ibc;
1277 mutex_lock(&kvm->lock);
1278 if (kvm->created_vcpus) {
1282 proc = kzalloc(sizeof(*proc), GFP_KERNEL_ACCOUNT);
1287 if (!copy_from_user(proc, (void __user *)attr->addr,
1289 kvm->arch.model.cpuid = proc->cpuid;
1290 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1291 unblocked_ibc = sclp.ibc & 0xfff;
1292 if (lowest_ibc && proc->ibc) {
1293 if (proc->ibc > unblocked_ibc)
1294 kvm->arch.model.ibc = unblocked_ibc;
1295 else if (proc->ibc < lowest_ibc)
1296 kvm->arch.model.ibc = lowest_ibc;
1298 kvm->arch.model.ibc = proc->ibc;
1300 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1301 S390_ARCH_FAC_LIST_SIZE_BYTE);
1302 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1303 kvm->arch.model.ibc,
1304 kvm->arch.model.cpuid);
1305 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1306 kvm->arch.model.fac_list[0],
1307 kvm->arch.model.fac_list[1],
1308 kvm->arch.model.fac_list[2]);
1313 mutex_unlock(&kvm->lock);
1317 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1318 struct kvm_device_attr *attr)
1320 struct kvm_s390_vm_cpu_feat data;
1322 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1324 if (!bitmap_subset((unsigned long *) data.feat,
1325 kvm_s390_available_cpu_feat,
1326 KVM_S390_VM_CPU_FEAT_NR_BITS))
1329 mutex_lock(&kvm->lock);
1330 if (kvm->created_vcpus) {
1331 mutex_unlock(&kvm->lock);
1334 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1335 KVM_S390_VM_CPU_FEAT_NR_BITS);
1336 mutex_unlock(&kvm->lock);
1337 VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1344 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1345 struct kvm_device_attr *attr)
1347 mutex_lock(&kvm->lock);
1348 if (kvm->created_vcpus) {
1349 mutex_unlock(&kvm->lock);
1353 if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1354 sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1355 mutex_unlock(&kvm->lock);
1358 mutex_unlock(&kvm->lock);
1360 VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1361 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1362 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1363 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1364 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1365 VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1366 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1367 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1368 VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1369 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1370 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1371 VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1372 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1373 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1374 VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx",
1375 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1376 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1377 VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1378 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1379 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1380 VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1381 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1382 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1383 VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1384 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1385 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1386 VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1387 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1388 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1389 VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1390 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1391 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1392 VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1393 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1394 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1395 VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1396 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1397 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1398 VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1399 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1400 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1401 VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1402 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1403 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1404 VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1405 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1406 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1407 VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1408 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1409 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1410 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1411 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1412 VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1413 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1414 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1415 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1416 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1421 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1425 switch (attr->attr) {
1426 case KVM_S390_VM_CPU_PROCESSOR:
1427 ret = kvm_s390_set_processor(kvm, attr);
1429 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1430 ret = kvm_s390_set_processor_feat(kvm, attr);
1432 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1433 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1439 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1441 struct kvm_s390_vm_cpu_processor *proc;
1444 proc = kzalloc(sizeof(*proc), GFP_KERNEL_ACCOUNT);
1449 proc->cpuid = kvm->arch.model.cpuid;
1450 proc->ibc = kvm->arch.model.ibc;
1451 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1452 S390_ARCH_FAC_LIST_SIZE_BYTE);
1453 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1454 kvm->arch.model.ibc,
1455 kvm->arch.model.cpuid);
1456 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1457 kvm->arch.model.fac_list[0],
1458 kvm->arch.model.fac_list[1],
1459 kvm->arch.model.fac_list[2]);
1460 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1467 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1469 struct kvm_s390_vm_cpu_machine *mach;
1472 mach = kzalloc(sizeof(*mach), GFP_KERNEL_ACCOUNT);
1477 get_cpu_id((struct cpuid *) &mach->cpuid);
1478 mach->ibc = sclp.ibc;
1479 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1480 S390_ARCH_FAC_LIST_SIZE_BYTE);
1481 memcpy((unsigned long *)&mach->fac_list, stfle_fac_list,
1482 sizeof(stfle_fac_list));
1483 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1484 kvm->arch.model.ibc,
1485 kvm->arch.model.cpuid);
1486 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1490 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1494 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1501 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1502 struct kvm_device_attr *attr)
1504 struct kvm_s390_vm_cpu_feat data;
1506 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1507 KVM_S390_VM_CPU_FEAT_NR_BITS);
1508 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1510 VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1517 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1518 struct kvm_device_attr *attr)
1520 struct kvm_s390_vm_cpu_feat data;
1522 bitmap_copy((unsigned long *) data.feat,
1523 kvm_s390_available_cpu_feat,
1524 KVM_S390_VM_CPU_FEAT_NR_BITS);
1525 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1527 VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1534 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1535 struct kvm_device_attr *attr)
1537 if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1538 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1541 VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1542 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1543 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1544 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1545 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1546 VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1547 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1548 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1549 VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1550 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1551 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1552 VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1553 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1554 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1555 VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx",
1556 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1557 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1558 VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1559 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1560 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1561 VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1562 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1563 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1564 VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1565 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1566 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1567 VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1568 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1569 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1570 VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1571 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1572 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1573 VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1574 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1575 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1576 VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1577 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1578 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1579 VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1580 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1581 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1582 VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1583 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1584 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1585 VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1586 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1587 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1588 VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1589 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1590 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1591 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1592 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1593 VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1594 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1595 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1596 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1597 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1602 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1603 struct kvm_device_attr *attr)
1605 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1606 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1609 VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1610 ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1611 ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1612 ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1613 ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1614 VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx",
1615 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1616 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1617 VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx",
1618 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1619 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1620 VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx",
1621 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1622 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1623 VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx",
1624 ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1625 ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1626 VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx",
1627 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1628 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1629 VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx",
1630 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1631 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1632 VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx",
1633 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1634 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1635 VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx",
1636 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1637 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1638 VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx",
1639 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1640 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1641 VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx",
1642 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1643 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1644 VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx",
1645 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1646 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1647 VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx",
1648 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1649 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1650 VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx",
1651 ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1652 ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
1653 VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx",
1654 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1655 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
1656 VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1657 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1658 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1659 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1660 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
1661 VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1662 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1663 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1664 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1665 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
1670 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1674 switch (attr->attr) {
1675 case KVM_S390_VM_CPU_PROCESSOR:
1676 ret = kvm_s390_get_processor(kvm, attr);
1678 case KVM_S390_VM_CPU_MACHINE:
1679 ret = kvm_s390_get_machine(kvm, attr);
1681 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1682 ret = kvm_s390_get_processor_feat(kvm, attr);
1684 case KVM_S390_VM_CPU_MACHINE_FEAT:
1685 ret = kvm_s390_get_machine_feat(kvm, attr);
1687 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1688 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1690 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1691 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1697 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1701 switch (attr->group) {
1702 case KVM_S390_VM_MEM_CTRL:
1703 ret = kvm_s390_set_mem_control(kvm, attr);
1705 case KVM_S390_VM_TOD:
1706 ret = kvm_s390_set_tod(kvm, attr);
1708 case KVM_S390_VM_CPU_MODEL:
1709 ret = kvm_s390_set_cpu_model(kvm, attr);
1711 case KVM_S390_VM_CRYPTO:
1712 ret = kvm_s390_vm_set_crypto(kvm, attr);
1714 case KVM_S390_VM_MIGRATION:
1715 ret = kvm_s390_vm_set_migration(kvm, attr);
1725 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1729 switch (attr->group) {
1730 case KVM_S390_VM_MEM_CTRL:
1731 ret = kvm_s390_get_mem_control(kvm, attr);
1733 case KVM_S390_VM_TOD:
1734 ret = kvm_s390_get_tod(kvm, attr);
1736 case KVM_S390_VM_CPU_MODEL:
1737 ret = kvm_s390_get_cpu_model(kvm, attr);
1739 case KVM_S390_VM_MIGRATION:
1740 ret = kvm_s390_vm_get_migration(kvm, attr);
1750 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1754 switch (attr->group) {
1755 case KVM_S390_VM_MEM_CTRL:
1756 switch (attr->attr) {
1757 case KVM_S390_VM_MEM_ENABLE_CMMA:
1758 case KVM_S390_VM_MEM_CLR_CMMA:
1759 ret = sclp.has_cmma ? 0 : -ENXIO;
1761 case KVM_S390_VM_MEM_LIMIT_SIZE:
1769 case KVM_S390_VM_TOD:
1770 switch (attr->attr) {
1771 case KVM_S390_VM_TOD_LOW:
1772 case KVM_S390_VM_TOD_HIGH:
1780 case KVM_S390_VM_CPU_MODEL:
1781 switch (attr->attr) {
1782 case KVM_S390_VM_CPU_PROCESSOR:
1783 case KVM_S390_VM_CPU_MACHINE:
1784 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1785 case KVM_S390_VM_CPU_MACHINE_FEAT:
1786 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1787 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1795 case KVM_S390_VM_CRYPTO:
1796 switch (attr->attr) {
1797 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1798 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1799 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1800 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1803 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1804 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1805 ret = ap_instructions_available() ? 0 : -ENXIO;
1812 case KVM_S390_VM_MIGRATION:
1823 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1827 int srcu_idx, i, r = 0;
1829 if (args->flags != 0)
1832 /* Is this guest using storage keys? */
1833 if (!mm_uses_skeys(current->mm))
1834 return KVM_S390_GET_SKEYS_NONE;
1836 /* Enforce sane limit on memory allocation */
1837 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1840 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL_ACCOUNT);
1844 mmap_read_lock(current->mm);
1845 srcu_idx = srcu_read_lock(&kvm->srcu);
1846 for (i = 0; i < args->count; i++) {
1847 hva = gfn_to_hva(kvm, args->start_gfn + i);
1848 if (kvm_is_error_hva(hva)) {
1853 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1857 srcu_read_unlock(&kvm->srcu, srcu_idx);
1858 mmap_read_unlock(current->mm);
1861 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1862 sizeof(uint8_t) * args->count);
1871 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1875 int srcu_idx, i, r = 0;
1878 if (args->flags != 0)
1881 /* Enforce sane limit on memory allocation */
1882 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1885 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL_ACCOUNT);
1889 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1890 sizeof(uint8_t) * args->count);
1896 /* Enable storage key handling for the guest */
1897 r = s390_enable_skey();
1902 mmap_read_lock(current->mm);
1903 srcu_idx = srcu_read_lock(&kvm->srcu);
1904 while (i < args->count) {
1906 hva = gfn_to_hva(kvm, args->start_gfn + i);
1907 if (kvm_is_error_hva(hva)) {
1912 /* Lowest order bit is reserved */
1913 if (keys[i] & 0x01) {
1918 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1920 r = fixup_user_fault(current->mm, hva,
1921 FAULT_FLAG_WRITE, &unlocked);
1928 srcu_read_unlock(&kvm->srcu, srcu_idx);
1929 mmap_read_unlock(current->mm);
1936 * Base address and length must be sent at the start of each block, therefore
1937 * it's cheaper to send some clean data, as long as it's less than the size of
1940 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1941 /* for consistency */
1942 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1945 * Similar to gfn_to_memslot, but returns the index of a memslot also when the
1946 * address falls in a hole. In that case the index of one of the memslots
1947 * bordering the hole is returned.
1949 static int gfn_to_memslot_approx(struct kvm_memslots *slots, gfn_t gfn)
1951 int start = 0, end = slots->used_slots;
1952 int slot = atomic_read(&slots->last_used_slot);
1953 struct kvm_memory_slot *memslots = slots->memslots;
1955 if (gfn >= memslots[slot].base_gfn &&
1956 gfn < memslots[slot].base_gfn + memslots[slot].npages)
1959 while (start < end) {
1960 slot = start + (end - start) / 2;
1962 if (gfn >= memslots[slot].base_gfn)
1968 if (start >= slots->used_slots)
1969 return slots->used_slots - 1;
1971 if (gfn >= memslots[start].base_gfn &&
1972 gfn < memslots[start].base_gfn + memslots[start].npages) {
1973 atomic_set(&slots->last_used_slot, start);
1979 static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1980 u8 *res, unsigned long bufsize)
1982 unsigned long pgstev, hva, cur_gfn = args->start_gfn;
1985 while (args->count < bufsize) {
1986 hva = gfn_to_hva(kvm, cur_gfn);
1988 * We return an error if the first value was invalid, but we
1989 * return successfully if at least one value was copied.
1991 if (kvm_is_error_hva(hva))
1992 return args->count ? 0 : -EFAULT;
1993 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
1995 res[args->count++] = (pgstev >> 24) & 0x43;
2002 static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
2003 unsigned long cur_gfn)
2005 int slotidx = gfn_to_memslot_approx(slots, cur_gfn);
2006 struct kvm_memory_slot *ms = slots->memslots + slotidx;
2007 unsigned long ofs = cur_gfn - ms->base_gfn;
2009 if (ms->base_gfn + ms->npages <= cur_gfn) {
2011 /* If we are above the highest slot, wrap around */
2013 slotidx = slots->used_slots - 1;
2015 ms = slots->memslots + slotidx;
2018 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
2019 while ((slotidx > 0) && (ofs >= ms->npages)) {
2021 ms = slots->memslots + slotidx;
2022 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, 0);
2024 return ms->base_gfn + ofs;
2027 static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
2028 u8 *res, unsigned long bufsize)
2030 unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
2031 struct kvm_memslots *slots = kvm_memslots(kvm);
2032 struct kvm_memory_slot *ms;
2034 if (unlikely(!slots->used_slots))
2037 cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
2038 ms = gfn_to_memslot(kvm, cur_gfn);
2040 args->start_gfn = cur_gfn;
2043 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2044 mem_end = slots->memslots[0].base_gfn + slots->memslots[0].npages;
2046 while (args->count < bufsize) {
2047 hva = gfn_to_hva(kvm, cur_gfn);
2048 if (kvm_is_error_hva(hva))
2050 /* Decrement only if we actually flipped the bit to 0 */
2051 if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
2052 atomic64_dec(&kvm->arch.cmma_dirty_pages);
2053 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
2055 /* Save the value */
2056 res[args->count++] = (pgstev >> 24) & 0x43;
2057 /* If the next bit is too far away, stop. */
2058 if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
2060 /* If we reached the previous "next", find the next one */
2061 if (cur_gfn == next_gfn)
2062 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2063 /* Reached the end of memory or of the buffer, stop */
2064 if ((next_gfn >= mem_end) ||
2065 (next_gfn - args->start_gfn >= bufsize))
2068 /* Reached the end of the current memslot, take the next one. */
2069 if (cur_gfn - ms->base_gfn >= ms->npages) {
2070 ms = gfn_to_memslot(kvm, cur_gfn);
2079 * This function searches for the next page with dirty CMMA attributes, and
2080 * saves the attributes in the buffer up to either the end of the buffer or
2081 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
2082 * no trailing clean bytes are saved.
2083 * In case no dirty bits were found, or if CMMA was not enabled or used, the
2084 * output buffer will indicate 0 as length.
2086 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
2087 struct kvm_s390_cmma_log *args)
2089 unsigned long bufsize;
2090 int srcu_idx, peek, ret;
2093 if (!kvm->arch.use_cmma)
2095 /* Invalid/unsupported flags were specified */
2096 if (args->flags & ~KVM_S390_CMMA_PEEK)
2098 /* Migration mode query, and we are not doing a migration */
2099 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
2100 if (!peek && !kvm->arch.migration_mode)
2102 /* CMMA is disabled or was not used, or the buffer has length zero */
2103 bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
2104 if (!bufsize || !kvm->mm->context.uses_cmm) {
2105 memset(args, 0, sizeof(*args));
2108 /* We are not peeking, and there are no dirty pages */
2109 if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
2110 memset(args, 0, sizeof(*args));
2114 values = vmalloc(bufsize);
2118 mmap_read_lock(kvm->mm);
2119 srcu_idx = srcu_read_lock(&kvm->srcu);
2121 ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
2123 ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
2124 srcu_read_unlock(&kvm->srcu, srcu_idx);
2125 mmap_read_unlock(kvm->mm);
2127 if (kvm->arch.migration_mode)
2128 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
2130 args->remaining = 0;
2132 if (copy_to_user((void __user *)args->values, values, args->count))
2140 * This function sets the CMMA attributes for the given pages. If the input
2141 * buffer has zero length, no action is taken, otherwise the attributes are
2142 * set and the mm->context.uses_cmm flag is set.
2144 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
2145 const struct kvm_s390_cmma_log *args)
2147 unsigned long hva, mask, pgstev, i;
2149 int srcu_idx, r = 0;
2153 if (!kvm->arch.use_cmma)
2155 /* invalid/unsupported flags */
2156 if (args->flags != 0)
2158 /* Enforce sane limit on memory allocation */
2159 if (args->count > KVM_S390_CMMA_SIZE_MAX)
2162 if (args->count == 0)
2165 bits = vmalloc(array_size(sizeof(*bits), args->count));
2169 r = copy_from_user(bits, (void __user *)args->values, args->count);
2175 mmap_read_lock(kvm->mm);
2176 srcu_idx = srcu_read_lock(&kvm->srcu);
2177 for (i = 0; i < args->count; i++) {
2178 hva = gfn_to_hva(kvm, args->start_gfn + i);
2179 if (kvm_is_error_hva(hva)) {
2185 pgstev = pgstev << 24;
2186 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
2187 set_pgste_bits(kvm->mm, hva, mask, pgstev);
2189 srcu_read_unlock(&kvm->srcu, srcu_idx);
2190 mmap_read_unlock(kvm->mm);
2192 if (!kvm->mm->context.uses_cmm) {
2193 mmap_write_lock(kvm->mm);
2194 kvm->mm->context.uses_cmm = 1;
2195 mmap_write_unlock(kvm->mm);
2202 static int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rcp, u16 *rrcp)
2204 struct kvm_vcpu *vcpu;
2210 * We ignore failures and try to destroy as many CPUs as possible.
2211 * At the same time we must not free the assigned resources when
2212 * this fails, as the ultravisor has still access to that memory.
2213 * So kvm_s390_pv_destroy_cpu can leave a "wanted" memory leak
2215 * We want to return the first failure rc and rrc, though.
2217 kvm_for_each_vcpu(i, vcpu, kvm) {
2218 mutex_lock(&vcpu->mutex);
2219 if (kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc) && !ret) {
2224 mutex_unlock(&vcpu->mutex);
2229 static int kvm_s390_cpus_to_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
2234 struct kvm_vcpu *vcpu;
2236 kvm_for_each_vcpu(i, vcpu, kvm) {
2237 mutex_lock(&vcpu->mutex);
2238 r = kvm_s390_pv_create_cpu(vcpu, rc, rrc);
2239 mutex_unlock(&vcpu->mutex);
2244 kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
2248 static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
2252 void __user *argp = (void __user *)cmd->data;
2255 case KVM_PV_ENABLE: {
2257 if (kvm_s390_pv_is_protected(kvm))
2261 * FMT 4 SIE needs esca. As we never switch back to bsca from
2262 * esca, we need no cleanup in the error cases below
2264 r = sca_switch_to_extended(kvm);
2268 mmap_write_lock(current->mm);
2269 r = gmap_mark_unmergeable();
2270 mmap_write_unlock(current->mm);
2274 r = kvm_s390_pv_init_vm(kvm, &cmd->rc, &cmd->rrc);
2278 r = kvm_s390_cpus_to_pv(kvm, &cmd->rc, &cmd->rrc);
2280 kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy);
2282 /* we need to block service interrupts from now on */
2283 set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2286 case KVM_PV_DISABLE: {
2288 if (!kvm_s390_pv_is_protected(kvm))
2291 r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
2293 * If a CPU could not be destroyed, destroy VM will also fail.
2294 * There is no point in trying to destroy it. Instead return
2295 * the rc and rrc from the first CPU that failed destroying.
2299 r = kvm_s390_pv_deinit_vm(kvm, &cmd->rc, &cmd->rrc);
2301 /* no need to block service interrupts any more */
2302 clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2305 case KVM_PV_SET_SEC_PARMS: {
2306 struct kvm_s390_pv_sec_parm parms = {};
2310 if (!kvm_s390_pv_is_protected(kvm))
2314 if (copy_from_user(&parms, argp, sizeof(parms)))
2317 /* Currently restricted to 8KB */
2319 if (parms.length > PAGE_SIZE * 2)
2323 hdr = vmalloc(parms.length);
2328 if (!copy_from_user(hdr, (void __user *)parms.origin,
2330 r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length,
2331 &cmd->rc, &cmd->rrc);
2336 case KVM_PV_UNPACK: {
2337 struct kvm_s390_pv_unp unp = {};
2340 if (!kvm_s390_pv_is_protected(kvm) || !mm_is_protected(kvm->mm))
2344 if (copy_from_user(&unp, argp, sizeof(unp)))
2347 r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak,
2348 &cmd->rc, &cmd->rrc);
2351 case KVM_PV_VERIFY: {
2353 if (!kvm_s390_pv_is_protected(kvm))
2356 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2357 UVC_CMD_VERIFY_IMG, &cmd->rc, &cmd->rrc);
2358 KVM_UV_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x", cmd->rc,
2362 case KVM_PV_PREP_RESET: {
2364 if (!kvm_s390_pv_is_protected(kvm))
2367 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2368 UVC_CMD_PREPARE_RESET, &cmd->rc, &cmd->rrc);
2369 KVM_UV_EVENT(kvm, 3, "PROTVIRT PREP RESET: rc %x rrc %x",
2373 case KVM_PV_UNSHARE_ALL: {
2375 if (!kvm_s390_pv_is_protected(kvm))
2378 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2379 UVC_CMD_SET_UNSHARE_ALL, &cmd->rc, &cmd->rrc);
2380 KVM_UV_EVENT(kvm, 3, "PROTVIRT UNSHARE: rc %x rrc %x",
2390 long kvm_arch_vm_ioctl(struct file *filp,
2391 unsigned int ioctl, unsigned long arg)
2393 struct kvm *kvm = filp->private_data;
2394 void __user *argp = (void __user *)arg;
2395 struct kvm_device_attr attr;
2399 case KVM_S390_INTERRUPT: {
2400 struct kvm_s390_interrupt s390int;
2403 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2405 r = kvm_s390_inject_vm(kvm, &s390int);
2408 case KVM_CREATE_IRQCHIP: {
2409 struct kvm_irq_routing_entry routing;
2412 if (kvm->arch.use_irqchip) {
2413 /* Set up dummy routing. */
2414 memset(&routing, 0, sizeof(routing));
2415 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
2419 case KVM_SET_DEVICE_ATTR: {
2421 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2423 r = kvm_s390_vm_set_attr(kvm, &attr);
2426 case KVM_GET_DEVICE_ATTR: {
2428 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2430 r = kvm_s390_vm_get_attr(kvm, &attr);
2433 case KVM_HAS_DEVICE_ATTR: {
2435 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2437 r = kvm_s390_vm_has_attr(kvm, &attr);
2440 case KVM_S390_GET_SKEYS: {
2441 struct kvm_s390_skeys args;
2444 if (copy_from_user(&args, argp,
2445 sizeof(struct kvm_s390_skeys)))
2447 r = kvm_s390_get_skeys(kvm, &args);
2450 case KVM_S390_SET_SKEYS: {
2451 struct kvm_s390_skeys args;
2454 if (copy_from_user(&args, argp,
2455 sizeof(struct kvm_s390_skeys)))
2457 r = kvm_s390_set_skeys(kvm, &args);
2460 case KVM_S390_GET_CMMA_BITS: {
2461 struct kvm_s390_cmma_log args;
2464 if (copy_from_user(&args, argp, sizeof(args)))
2466 mutex_lock(&kvm->slots_lock);
2467 r = kvm_s390_get_cmma_bits(kvm, &args);
2468 mutex_unlock(&kvm->slots_lock);
2470 r = copy_to_user(argp, &args, sizeof(args));
2476 case KVM_S390_SET_CMMA_BITS: {
2477 struct kvm_s390_cmma_log args;
2480 if (copy_from_user(&args, argp, sizeof(args)))
2482 mutex_lock(&kvm->slots_lock);
2483 r = kvm_s390_set_cmma_bits(kvm, &args);
2484 mutex_unlock(&kvm->slots_lock);
2487 case KVM_S390_PV_COMMAND: {
2488 struct kvm_pv_cmd args;
2490 /* protvirt means user sigp */
2491 kvm->arch.user_cpu_state_ctrl = 1;
2493 if (!is_prot_virt_host()) {
2497 if (copy_from_user(&args, argp, sizeof(args))) {
2505 mutex_lock(&kvm->lock);
2506 r = kvm_s390_handle_pv(kvm, &args);
2507 mutex_unlock(&kvm->lock);
2508 if (copy_to_user(argp, &args, sizeof(args))) {
2521 static int kvm_s390_apxa_installed(void)
2523 struct ap_config_info info;
2525 if (ap_instructions_available()) {
2526 if (ap_qci(&info) == 0)
2534 * The format of the crypto control block (CRYCB) is specified in the 3 low
2535 * order bits of the CRYCB designation (CRYCBD) field as follows:
2536 * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
2537 * AP extended addressing (APXA) facility are installed.
2538 * Format 1: The APXA facility is not installed but the MSAX3 facility is.
2539 * Format 2: Both the APXA and MSAX3 facilities are installed
2541 static void kvm_s390_set_crycb_format(struct kvm *kvm)
2543 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
2545 /* Clear the CRYCB format bits - i.e., set format 0 by default */
2546 kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
2548 /* Check whether MSAX3 is installed */
2549 if (!test_kvm_facility(kvm, 76))
2552 if (kvm_s390_apxa_installed())
2553 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
2555 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
2559 * kvm_arch_crypto_set_masks
2561 * @kvm: pointer to the target guest's KVM struct containing the crypto masks
2563 * @apm: the mask identifying the accessible AP adapters
2564 * @aqm: the mask identifying the accessible AP domains
2565 * @adm: the mask identifying the accessible AP control domains
2567 * Set the masks that identify the adapters, domains and control domains to
2568 * which the KVM guest is granted access.
2570 * Note: The kvm->lock mutex must be locked by the caller before invoking this
2573 void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
2574 unsigned long *aqm, unsigned long *adm)
2576 struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
2578 kvm_s390_vcpu_block_all(kvm);
2580 switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
2581 case CRYCB_FORMAT2: /* APCB1 use 256 bits */
2582 memcpy(crycb->apcb1.apm, apm, 32);
2583 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
2584 apm[0], apm[1], apm[2], apm[3]);
2585 memcpy(crycb->apcb1.aqm, aqm, 32);
2586 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
2587 aqm[0], aqm[1], aqm[2], aqm[3]);
2588 memcpy(crycb->apcb1.adm, adm, 32);
2589 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
2590 adm[0], adm[1], adm[2], adm[3]);
2593 case CRYCB_FORMAT0: /* Fall through both use APCB0 */
2594 memcpy(crycb->apcb0.apm, apm, 8);
2595 memcpy(crycb->apcb0.aqm, aqm, 2);
2596 memcpy(crycb->apcb0.adm, adm, 2);
2597 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
2598 apm[0], *((unsigned short *)aqm),
2599 *((unsigned short *)adm));
2601 default: /* Can not happen */
2605 /* recreate the shadow crycb for each vcpu */
2606 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2607 kvm_s390_vcpu_unblock_all(kvm);
2609 EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
2612 * kvm_arch_crypto_clear_masks
2614 * @kvm: pointer to the target guest's KVM struct containing the crypto masks
2617 * Clear the masks that identify the adapters, domains and control domains to
2618 * which the KVM guest is granted access.
2620 * Note: The kvm->lock mutex must be locked by the caller before invoking this
2623 void kvm_arch_crypto_clear_masks(struct kvm *kvm)
2625 kvm_s390_vcpu_block_all(kvm);
2627 memset(&kvm->arch.crypto.crycb->apcb0, 0,
2628 sizeof(kvm->arch.crypto.crycb->apcb0));
2629 memset(&kvm->arch.crypto.crycb->apcb1, 0,
2630 sizeof(kvm->arch.crypto.crycb->apcb1));
2632 VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
2633 /* recreate the shadow crycb for each vcpu */
2634 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2635 kvm_s390_vcpu_unblock_all(kvm);
2637 EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
2639 static u64 kvm_s390_get_initial_cpuid(void)
2644 cpuid.version = 0xff;
2645 return *((u64 *) &cpuid);
2648 static void kvm_s390_crypto_init(struct kvm *kvm)
2650 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
2651 kvm_s390_set_crycb_format(kvm);
2652 init_rwsem(&kvm->arch.crypto.pqap_hook_rwsem);
2654 if (!test_kvm_facility(kvm, 76))
2657 /* Enable AES/DEA protected key functions by default */
2658 kvm->arch.crypto.aes_kw = 1;
2659 kvm->arch.crypto.dea_kw = 1;
2660 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
2661 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
2662 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
2663 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
2666 static void sca_dispose(struct kvm *kvm)
2668 if (kvm->arch.use_esca)
2669 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
2671 free_page((unsigned long)(kvm->arch.sca));
2672 kvm->arch.sca = NULL;
2675 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
2677 gfp_t alloc_flags = GFP_KERNEL_ACCOUNT;
2679 char debug_name[16];
2680 static unsigned long sca_offset;
2683 #ifdef CONFIG_KVM_S390_UCONTROL
2684 if (type & ~KVM_VM_S390_UCONTROL)
2686 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
2693 rc = s390_enable_sie();
2699 if (!sclp.has_64bscao)
2700 alloc_flags |= GFP_DMA;
2701 rwlock_init(&kvm->arch.sca_lock);
2702 /* start with basic SCA */
2703 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
2706 mutex_lock(&kvm_lock);
2708 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
2710 kvm->arch.sca = (struct bsca_block *)
2711 ((char *) kvm->arch.sca + sca_offset);
2712 mutex_unlock(&kvm_lock);
2714 sprintf(debug_name, "kvm-%u", current->pid);
2716 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
2720 BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
2721 kvm->arch.sie_page2 =
2722 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
2723 if (!kvm->arch.sie_page2)
2726 kvm->arch.sie_page2->kvm = kvm;
2727 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
2729 for (i = 0; i < kvm_s390_fac_size(); i++) {
2730 kvm->arch.model.fac_mask[i] = stfle_fac_list[i] &
2731 (kvm_s390_fac_base[i] |
2732 kvm_s390_fac_ext[i]);
2733 kvm->arch.model.fac_list[i] = stfle_fac_list[i] &
2734 kvm_s390_fac_base[i];
2736 kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
2738 /* we are always in czam mode - even on pre z14 machines */
2739 set_kvm_facility(kvm->arch.model.fac_mask, 138);
2740 set_kvm_facility(kvm->arch.model.fac_list, 138);
2741 /* we emulate STHYI in kvm */
2742 set_kvm_facility(kvm->arch.model.fac_mask, 74);
2743 set_kvm_facility(kvm->arch.model.fac_list, 74);
2744 if (MACHINE_HAS_TLB_GUEST) {
2745 set_kvm_facility(kvm->arch.model.fac_mask, 147);
2746 set_kvm_facility(kvm->arch.model.fac_list, 147);
2749 if (css_general_characteristics.aiv && test_facility(65))
2750 set_kvm_facility(kvm->arch.model.fac_mask, 65);
2752 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
2753 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
2755 kvm_s390_crypto_init(kvm);
2757 mutex_init(&kvm->arch.float_int.ais_lock);
2758 spin_lock_init(&kvm->arch.float_int.lock);
2759 for (i = 0; i < FIRQ_LIST_COUNT; i++)
2760 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
2761 init_waitqueue_head(&kvm->arch.ipte_wq);
2762 mutex_init(&kvm->arch.ipte_mutex);
2764 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
2765 VM_EVENT(kvm, 3, "vm created with type %lu", type);
2767 if (type & KVM_VM_S390_UCONTROL) {
2768 kvm->arch.gmap = NULL;
2769 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
2771 if (sclp.hamax == U64_MAX)
2772 kvm->arch.mem_limit = TASK_SIZE_MAX;
2774 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
2776 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
2777 if (!kvm->arch.gmap)
2779 kvm->arch.gmap->private = kvm;
2780 kvm->arch.gmap->pfault_enabled = 0;
2783 kvm->arch.use_pfmfi = sclp.has_pfmfi;
2784 kvm->arch.use_skf = sclp.has_skey;
2785 spin_lock_init(&kvm->arch.start_stop_lock);
2786 kvm_s390_vsie_init(kvm);
2788 kvm_s390_gisa_init(kvm);
2789 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
2793 free_page((unsigned long)kvm->arch.sie_page2);
2794 debug_unregister(kvm->arch.dbf);
2796 KVM_EVENT(3, "creation of vm failed: %d", rc);
2800 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2804 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
2805 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
2806 kvm_s390_clear_local_irqs(vcpu);
2807 kvm_clear_async_pf_completion_queue(vcpu);
2808 if (!kvm_is_ucontrol(vcpu->kvm))
2811 if (kvm_is_ucontrol(vcpu->kvm))
2812 gmap_remove(vcpu->arch.gmap);
2814 if (vcpu->kvm->arch.use_cmma)
2815 kvm_s390_vcpu_unsetup_cmma(vcpu);
2816 /* We can not hold the vcpu mutex here, we are already dying */
2817 if (kvm_s390_pv_cpu_get_handle(vcpu))
2818 kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc);
2819 free_page((unsigned long)(vcpu->arch.sie_block));
2822 static void kvm_free_vcpus(struct kvm *kvm)
2825 struct kvm_vcpu *vcpu;
2827 kvm_for_each_vcpu(i, vcpu, kvm)
2828 kvm_vcpu_destroy(vcpu);
2830 mutex_lock(&kvm->lock);
2831 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2832 kvm->vcpus[i] = NULL;
2834 atomic_set(&kvm->online_vcpus, 0);
2835 mutex_unlock(&kvm->lock);
2838 void kvm_arch_destroy_vm(struct kvm *kvm)
2842 kvm_free_vcpus(kvm);
2844 kvm_s390_gisa_destroy(kvm);
2846 * We are already at the end of life and kvm->lock is not taken.
2847 * This is ok as the file descriptor is closed by now and nobody
2848 * can mess with the pv state. To avoid lockdep_assert_held from
2849 * complaining we do not use kvm_s390_pv_is_protected.
2851 if (kvm_s390_pv_get_handle(kvm))
2852 kvm_s390_pv_deinit_vm(kvm, &rc, &rrc);
2853 debug_unregister(kvm->arch.dbf);
2854 free_page((unsigned long)kvm->arch.sie_page2);
2855 if (!kvm_is_ucontrol(kvm))
2856 gmap_remove(kvm->arch.gmap);
2857 kvm_s390_destroy_adapters(kvm);
2858 kvm_s390_clear_float_irqs(kvm);
2859 kvm_s390_vsie_destroy(kvm);
2860 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
2863 /* Section: vcpu related */
2864 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2866 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
2867 if (!vcpu->arch.gmap)
2869 vcpu->arch.gmap->private = vcpu->kvm;
2874 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2876 if (!kvm_s390_use_sca_entries())
2878 read_lock(&vcpu->kvm->arch.sca_lock);
2879 if (vcpu->kvm->arch.use_esca) {
2880 struct esca_block *sca = vcpu->kvm->arch.sca;
2882 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2883 sca->cpu[vcpu->vcpu_id].sda = 0;
2885 struct bsca_block *sca = vcpu->kvm->arch.sca;
2887 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2888 sca->cpu[vcpu->vcpu_id].sda = 0;
2890 read_unlock(&vcpu->kvm->arch.sca_lock);
2893 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
2895 if (!kvm_s390_use_sca_entries()) {
2896 struct bsca_block *sca = vcpu->kvm->arch.sca;
2898 /* we still need the basic sca for the ipte control */
2899 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2900 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2903 read_lock(&vcpu->kvm->arch.sca_lock);
2904 if (vcpu->kvm->arch.use_esca) {
2905 struct esca_block *sca = vcpu->kvm->arch.sca;
2907 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2908 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2909 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
2910 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2911 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2913 struct bsca_block *sca = vcpu->kvm->arch.sca;
2915 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2916 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2917 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2918 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2920 read_unlock(&vcpu->kvm->arch.sca_lock);
2923 /* Basic SCA to Extended SCA data copy routines */
2924 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2927 d->sigp_ctrl.c = s->sigp_ctrl.c;
2928 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2931 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2935 d->ipte_control = s->ipte_control;
2937 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2938 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2941 static int sca_switch_to_extended(struct kvm *kvm)
2943 struct bsca_block *old_sca = kvm->arch.sca;
2944 struct esca_block *new_sca;
2945 struct kvm_vcpu *vcpu;
2946 unsigned int vcpu_idx;
2949 if (kvm->arch.use_esca)
2952 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL_ACCOUNT | __GFP_ZERO);
2956 scaoh = (u32)((u64)(new_sca) >> 32);
2957 scaol = (u32)(u64)(new_sca) & ~0x3fU;
2959 kvm_s390_vcpu_block_all(kvm);
2960 write_lock(&kvm->arch.sca_lock);
2962 sca_copy_b_to_e(new_sca, old_sca);
2964 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2965 vcpu->arch.sie_block->scaoh = scaoh;
2966 vcpu->arch.sie_block->scaol = scaol;
2967 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2969 kvm->arch.sca = new_sca;
2970 kvm->arch.use_esca = 1;
2972 write_unlock(&kvm->arch.sca_lock);
2973 kvm_s390_vcpu_unblock_all(kvm);
2975 free_page((unsigned long)old_sca);
2977 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2978 old_sca, kvm->arch.sca);
2982 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2986 if (!kvm_s390_use_sca_entries()) {
2987 if (id < KVM_MAX_VCPUS)
2991 if (id < KVM_S390_BSCA_CPU_SLOTS)
2993 if (!sclp.has_esca || !sclp.has_64bscao)
2996 mutex_lock(&kvm->lock);
2997 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2998 mutex_unlock(&kvm->lock);
3000 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
3003 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
3004 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3006 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
3007 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3008 vcpu->arch.cputm_start = get_tod_clock_fast();
3009 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3012 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
3013 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3015 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
3016 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3017 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3018 vcpu->arch.cputm_start = 0;
3019 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3022 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
3023 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3025 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
3026 vcpu->arch.cputm_enabled = true;
3027 __start_cpu_timer_accounting(vcpu);
3030 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
3031 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3033 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
3034 __stop_cpu_timer_accounting(vcpu);
3035 vcpu->arch.cputm_enabled = false;
3038 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3040 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3041 __enable_cpu_timer_accounting(vcpu);
3045 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3047 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3048 __disable_cpu_timer_accounting(vcpu);
3052 /* set the cpu timer - may only be called from the VCPU thread itself */
3053 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
3055 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3056 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3057 if (vcpu->arch.cputm_enabled)
3058 vcpu->arch.cputm_start = get_tod_clock_fast();
3059 vcpu->arch.sie_block->cputm = cputm;
3060 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3064 /* update and get the cpu timer - can also be called from other VCPU threads */
3065 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
3070 if (unlikely(!vcpu->arch.cputm_enabled))
3071 return vcpu->arch.sie_block->cputm;
3073 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3075 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
3077 * If the writer would ever execute a read in the critical
3078 * section, e.g. in irq context, we have a deadlock.
3080 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
3081 value = vcpu->arch.sie_block->cputm;
3082 /* if cputm_start is 0, accounting is being started/stopped */
3083 if (likely(vcpu->arch.cputm_start))
3084 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3085 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
3090 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
3093 gmap_enable(vcpu->arch.enabled_gmap);
3094 kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
3095 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3096 __start_cpu_timer_accounting(vcpu);
3100 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
3103 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3104 __stop_cpu_timer_accounting(vcpu);
3105 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
3106 vcpu->arch.enabled_gmap = gmap_get_enabled();
3107 gmap_disable(vcpu->arch.enabled_gmap);
3111 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
3113 mutex_lock(&vcpu->kvm->lock);
3115 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
3116 vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
3118 mutex_unlock(&vcpu->kvm->lock);
3119 if (!kvm_is_ucontrol(vcpu->kvm)) {
3120 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
3123 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
3124 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3125 /* make vcpu_load load the right gmap on the first trigger */
3126 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
3129 static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
3131 if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
3132 test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
3137 static bool kvm_has_pckmo_ecc(struct kvm *kvm)
3139 /* At least one ECC subfunction must be present */
3140 return kvm_has_pckmo_subfunc(kvm, 32) ||
3141 kvm_has_pckmo_subfunc(kvm, 33) ||
3142 kvm_has_pckmo_subfunc(kvm, 34) ||
3143 kvm_has_pckmo_subfunc(kvm, 40) ||
3144 kvm_has_pckmo_subfunc(kvm, 41);
3148 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
3151 * If the AP instructions are not being interpreted and the MSAX3
3152 * facility is not configured for the guest, there is nothing to set up.
3154 if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
3157 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
3158 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
3159 vcpu->arch.sie_block->eca &= ~ECA_APIE;
3160 vcpu->arch.sie_block->ecd &= ~ECD_ECC;
3162 if (vcpu->kvm->arch.crypto.apie)
3163 vcpu->arch.sie_block->eca |= ECA_APIE;
3165 /* Set up protected key support */
3166 if (vcpu->kvm->arch.crypto.aes_kw) {
3167 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
3168 /* ecc is also wrapped with AES key */
3169 if (kvm_has_pckmo_ecc(vcpu->kvm))
3170 vcpu->arch.sie_block->ecd |= ECD_ECC;
3173 if (vcpu->kvm->arch.crypto.dea_kw)
3174 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
3177 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
3179 free_page(vcpu->arch.sie_block->cbrlo);
3180 vcpu->arch.sie_block->cbrlo = 0;
3183 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
3185 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL_ACCOUNT);
3186 if (!vcpu->arch.sie_block->cbrlo)
3191 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
3193 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
3195 vcpu->arch.sie_block->ibc = model->ibc;
3196 if (test_kvm_facility(vcpu->kvm, 7))
3197 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
3200 static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
3205 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
3209 if (test_kvm_facility(vcpu->kvm, 78))
3210 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
3211 else if (test_kvm_facility(vcpu->kvm, 8))
3212 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
3214 kvm_s390_vcpu_setup_model(vcpu);
3216 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
3217 if (MACHINE_HAS_ESOP)
3218 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
3219 if (test_kvm_facility(vcpu->kvm, 9))
3220 vcpu->arch.sie_block->ecb |= ECB_SRSI;
3221 if (test_kvm_facility(vcpu->kvm, 73))
3222 vcpu->arch.sie_block->ecb |= ECB_TE;
3223 if (!kvm_is_ucontrol(vcpu->kvm))
3224 vcpu->arch.sie_block->ecb |= ECB_SPECI;
3226 if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
3227 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
3228 if (test_kvm_facility(vcpu->kvm, 130))
3229 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
3230 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
3232 vcpu->arch.sie_block->eca |= ECA_CEI;
3234 vcpu->arch.sie_block->eca |= ECA_IB;
3236 vcpu->arch.sie_block->eca |= ECA_SII;
3237 if (sclp.has_sigpif)
3238 vcpu->arch.sie_block->eca |= ECA_SIGPI;
3239 if (test_kvm_facility(vcpu->kvm, 129)) {
3240 vcpu->arch.sie_block->eca |= ECA_VX;
3241 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3243 if (test_kvm_facility(vcpu->kvm, 139))
3244 vcpu->arch.sie_block->ecd |= ECD_MEF;
3245 if (test_kvm_facility(vcpu->kvm, 156))
3246 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
3247 if (vcpu->arch.sie_block->gd) {
3248 vcpu->arch.sie_block->eca |= ECA_AIV;
3249 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3250 vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3252 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
3254 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
3257 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
3259 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
3261 if (vcpu->kvm->arch.use_cmma) {
3262 rc = kvm_s390_vcpu_setup_cmma(vcpu);
3266 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3267 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
3269 vcpu->arch.sie_block->hpid = HPID_KVM;
3271 kvm_s390_vcpu_crypto_setup(vcpu);
3273 mutex_lock(&vcpu->kvm->lock);
3274 if (kvm_s390_pv_is_protected(vcpu->kvm)) {
3275 rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc);
3277 kvm_s390_vcpu_unsetup_cmma(vcpu);
3279 mutex_unlock(&vcpu->kvm->lock);
3284 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
3286 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
3291 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
3293 struct sie_page *sie_page;
3296 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
3297 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL_ACCOUNT);
3301 vcpu->arch.sie_block = &sie_page->sie_block;
3302 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
3304 /* the real guest size will always be smaller than msl */
3305 vcpu->arch.sie_block->mso = 0;
3306 vcpu->arch.sie_block->msl = sclp.hamax;
3308 vcpu->arch.sie_block->icpua = vcpu->vcpu_id;
3309 spin_lock_init(&vcpu->arch.local_int.lock);
3310 vcpu->arch.sie_block->gd = (u32)(u64)vcpu->kvm->arch.gisa_int.origin;
3311 if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
3312 vcpu->arch.sie_block->gd |= GISA_FORMAT1;
3313 seqcount_init(&vcpu->arch.cputm_seqcount);
3315 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3316 kvm_clear_async_pf_completion_queue(vcpu);
3317 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
3324 kvm_s390_set_prefix(vcpu, 0);
3325 if (test_kvm_facility(vcpu->kvm, 64))
3326 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
3327 if (test_kvm_facility(vcpu->kvm, 82))
3328 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
3329 if (test_kvm_facility(vcpu->kvm, 133))
3330 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
3331 if (test_kvm_facility(vcpu->kvm, 156))
3332 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
3333 /* fprs can be synchronized via vrs, even if the guest has no vx. With
3334 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
3337 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
3339 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
3341 if (kvm_is_ucontrol(vcpu->kvm)) {
3342 rc = __kvm_ucontrol_vcpu_init(vcpu);
3344 goto out_free_sie_block;
3347 VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK",
3348 vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3349 trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3351 rc = kvm_s390_vcpu_setup(vcpu);
3353 goto out_ucontrol_uninit;
3356 out_ucontrol_uninit:
3357 if (kvm_is_ucontrol(vcpu->kvm))
3358 gmap_remove(vcpu->arch.gmap);
3360 free_page((unsigned long)(vcpu->arch.sie_block));
3364 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3366 clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
3367 return kvm_s390_vcpu_has_irq(vcpu, 0);
3370 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
3372 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
3375 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
3377 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3381 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
3383 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3386 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
3388 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3392 bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
3394 return atomic_read(&vcpu->arch.sie_block->prog20) &
3395 (PROG_BLOCK_SIE | PROG_REQUEST);
3398 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
3400 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3404 * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
3405 * If the CPU is not running (e.g. waiting as idle) the function will
3406 * return immediately. */
3407 void exit_sie(struct kvm_vcpu *vcpu)
3409 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
3410 kvm_s390_vsie_kick(vcpu);
3411 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
3415 /* Kick a guest cpu out of SIE to process a request synchronously */
3416 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
3418 kvm_make_request(req, vcpu);
3419 kvm_s390_vcpu_request(vcpu);
3422 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
3425 struct kvm *kvm = gmap->private;
3426 struct kvm_vcpu *vcpu;
3427 unsigned long prefix;
3430 if (gmap_is_shadow(gmap))
3432 if (start >= 1UL << 31)
3433 /* We are only interested in prefix pages */
3435 kvm_for_each_vcpu(i, vcpu, kvm) {
3436 /* match against both prefix pages */
3437 prefix = kvm_s390_get_prefix(vcpu);
3438 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
3439 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
3441 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
3446 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
3448 /* do not poll with more than halt_poll_max_steal percent of steal time */
3449 if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
3450 READ_ONCE(halt_poll_max_steal)) {
3451 vcpu->stat.halt_no_poll_steal++;
3457 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
3459 /* kvm common code refers to this, but never calls it */
3464 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
3465 struct kvm_one_reg *reg)
3470 case KVM_REG_S390_TODPR:
3471 r = put_user(vcpu->arch.sie_block->todpr,
3472 (u32 __user *)reg->addr);
3474 case KVM_REG_S390_EPOCHDIFF:
3475 r = put_user(vcpu->arch.sie_block->epoch,
3476 (u64 __user *)reg->addr);
3478 case KVM_REG_S390_CPU_TIMER:
3479 r = put_user(kvm_s390_get_cpu_timer(vcpu),
3480 (u64 __user *)reg->addr);
3482 case KVM_REG_S390_CLOCK_COMP:
3483 r = put_user(vcpu->arch.sie_block->ckc,
3484 (u64 __user *)reg->addr);
3486 case KVM_REG_S390_PFTOKEN:
3487 r = put_user(vcpu->arch.pfault_token,
3488 (u64 __user *)reg->addr);
3490 case KVM_REG_S390_PFCOMPARE:
3491 r = put_user(vcpu->arch.pfault_compare,
3492 (u64 __user *)reg->addr);
3494 case KVM_REG_S390_PFSELECT:
3495 r = put_user(vcpu->arch.pfault_select,
3496 (u64 __user *)reg->addr);
3498 case KVM_REG_S390_PP:
3499 r = put_user(vcpu->arch.sie_block->pp,
3500 (u64 __user *)reg->addr);
3502 case KVM_REG_S390_GBEA:
3503 r = put_user(vcpu->arch.sie_block->gbea,
3504 (u64 __user *)reg->addr);
3513 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
3514 struct kvm_one_reg *reg)
3520 case KVM_REG_S390_TODPR:
3521 r = get_user(vcpu->arch.sie_block->todpr,
3522 (u32 __user *)reg->addr);
3524 case KVM_REG_S390_EPOCHDIFF:
3525 r = get_user(vcpu->arch.sie_block->epoch,
3526 (u64 __user *)reg->addr);
3528 case KVM_REG_S390_CPU_TIMER:
3529 r = get_user(val, (u64 __user *)reg->addr);
3531 kvm_s390_set_cpu_timer(vcpu, val);
3533 case KVM_REG_S390_CLOCK_COMP:
3534 r = get_user(vcpu->arch.sie_block->ckc,
3535 (u64 __user *)reg->addr);
3537 case KVM_REG_S390_PFTOKEN:
3538 r = get_user(vcpu->arch.pfault_token,
3539 (u64 __user *)reg->addr);
3540 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3541 kvm_clear_async_pf_completion_queue(vcpu);
3543 case KVM_REG_S390_PFCOMPARE:
3544 r = get_user(vcpu->arch.pfault_compare,
3545 (u64 __user *)reg->addr);
3547 case KVM_REG_S390_PFSELECT:
3548 r = get_user(vcpu->arch.pfault_select,
3549 (u64 __user *)reg->addr);
3551 case KVM_REG_S390_PP:
3552 r = get_user(vcpu->arch.sie_block->pp,
3553 (u64 __user *)reg->addr);
3555 case KVM_REG_S390_GBEA:
3556 r = get_user(vcpu->arch.sie_block->gbea,
3557 (u64 __user *)reg->addr);
3566 static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
3568 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI;
3569 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3570 memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
3572 kvm_clear_async_pf_completion_queue(vcpu);
3573 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
3574 kvm_s390_vcpu_stop(vcpu);
3575 kvm_s390_clear_local_irqs(vcpu);
3578 static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
3580 /* Initial reset is a superset of the normal reset */
3581 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
3584 * This equals initial cpu reset in pop, but we don't switch to ESA.
3585 * We do not only reset the internal data, but also ...
3587 vcpu->arch.sie_block->gpsw.mask = 0;
3588 vcpu->arch.sie_block->gpsw.addr = 0;
3589 kvm_s390_set_prefix(vcpu, 0);
3590 kvm_s390_set_cpu_timer(vcpu, 0);
3591 vcpu->arch.sie_block->ckc = 0;
3592 memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr));
3593 vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK;
3594 vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK;
3596 /* ... the data in sync regs */
3597 memset(vcpu->run->s.regs.crs, 0, sizeof(vcpu->run->s.regs.crs));
3598 vcpu->run->s.regs.ckc = 0;
3599 vcpu->run->s.regs.crs[0] = CR0_INITIAL_MASK;
3600 vcpu->run->s.regs.crs[14] = CR14_INITIAL_MASK;
3601 vcpu->run->psw_addr = 0;
3602 vcpu->run->psw_mask = 0;
3603 vcpu->run->s.regs.todpr = 0;
3604 vcpu->run->s.regs.cputm = 0;
3605 vcpu->run->s.regs.ckc = 0;
3606 vcpu->run->s.regs.pp = 0;
3607 vcpu->run->s.regs.gbea = 1;
3608 vcpu->run->s.regs.fpc = 0;
3610 * Do not reset these registers in the protected case, as some of
3611 * them are overlayed and they are not accessible in this case
3614 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
3615 vcpu->arch.sie_block->gbea = 1;
3616 vcpu->arch.sie_block->pp = 0;
3617 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3618 vcpu->arch.sie_block->todpr = 0;
3622 static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
3624 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
3626 /* Clear reset is a superset of the initial reset */
3627 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3629 memset(®s->gprs, 0, sizeof(regs->gprs));
3630 memset(®s->vrs, 0, sizeof(regs->vrs));
3631 memset(®s->acrs, 0, sizeof(regs->acrs));
3632 memset(®s->gscb, 0, sizeof(regs->gscb));
3635 regs->etoken_extension = 0;
3638 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3641 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs));
3646 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3649 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
3654 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
3655 struct kvm_sregs *sregs)
3659 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
3660 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
3666 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
3667 struct kvm_sregs *sregs)
3671 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
3672 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
3678 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3684 if (test_fp_ctl(fpu->fpc)) {
3688 vcpu->run->s.regs.fpc = fpu->fpc;
3690 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
3691 (freg_t *) fpu->fprs);
3693 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
3700 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3704 /* make sure we have the latest values */
3707 convert_vx_to_fp((freg_t *) fpu->fprs,
3708 (__vector128 *) vcpu->run->s.regs.vrs);
3710 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
3711 fpu->fpc = vcpu->run->s.regs.fpc;
3717 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
3721 if (!is_vcpu_stopped(vcpu))
3724 vcpu->run->psw_mask = psw.mask;
3725 vcpu->run->psw_addr = psw.addr;
3730 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
3731 struct kvm_translation *tr)
3733 return -EINVAL; /* not implemented yet */
3736 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
3737 KVM_GUESTDBG_USE_HW_BP | \
3738 KVM_GUESTDBG_ENABLE)
3740 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
3741 struct kvm_guest_debug *dbg)
3747 vcpu->guest_debug = 0;
3748 kvm_s390_clear_bp_data(vcpu);
3750 if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
3754 if (!sclp.has_gpere) {
3759 if (dbg->control & KVM_GUESTDBG_ENABLE) {
3760 vcpu->guest_debug = dbg->control;
3761 /* enforce guest PER */
3762 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
3764 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
3765 rc = kvm_s390_import_bp_data(vcpu, dbg);
3767 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3768 vcpu->arch.guestdbg.last_bp = 0;
3772 vcpu->guest_debug = 0;
3773 kvm_s390_clear_bp_data(vcpu);
3774 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3782 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
3783 struct kvm_mp_state *mp_state)
3789 /* CHECK_STOP and LOAD are not supported yet */
3790 ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
3791 KVM_MP_STATE_OPERATING;
3797 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
3798 struct kvm_mp_state *mp_state)
3804 /* user space knows about this interface - let it control the state */
3805 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
3807 switch (mp_state->mp_state) {
3808 case KVM_MP_STATE_STOPPED:
3809 rc = kvm_s390_vcpu_stop(vcpu);
3811 case KVM_MP_STATE_OPERATING:
3812 rc = kvm_s390_vcpu_start(vcpu);
3814 case KVM_MP_STATE_LOAD:
3815 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
3819 rc = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR_LOAD);
3821 case KVM_MP_STATE_CHECK_STOP:
3822 fallthrough; /* CHECK_STOP and LOAD are not supported yet */
3831 static bool ibs_enabled(struct kvm_vcpu *vcpu)
3833 return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
3836 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
3839 kvm_s390_vcpu_request_handled(vcpu);
3840 if (!kvm_request_pending(vcpu))
3843 * We use MMU_RELOAD just to re-arm the ipte notifier for the
3844 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
3845 * This ensures that the ipte instruction for this request has
3846 * already finished. We might race against a second unmapper that
3847 * wants to set the blocking bit. Lets just retry the request loop.
3849 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
3851 rc = gmap_mprotect_notify(vcpu->arch.gmap,
3852 kvm_s390_get_prefix(vcpu),
3853 PAGE_SIZE * 2, PROT_WRITE);
3855 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
3861 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
3862 vcpu->arch.sie_block->ihcpu = 0xffff;
3866 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
3867 if (!ibs_enabled(vcpu)) {
3868 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
3869 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
3874 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
3875 if (ibs_enabled(vcpu)) {
3876 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
3877 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
3882 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
3883 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3887 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
3889 * Disable CMM virtualization; we will emulate the ESSA
3890 * instruction manually, in order to provide additional
3891 * functionalities needed for live migration.
3893 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
3897 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
3899 * Re-enable CMM virtualization if CMMA is available and
3900 * CMM has been used.
3902 if ((vcpu->kvm->arch.use_cmma) &&
3903 (vcpu->kvm->mm->context.uses_cmm))
3904 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
3908 /* nothing to do, just clear the request */
3909 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
3910 /* we left the vsie handler, nothing to do, just clear the request */
3911 kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
3916 static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
3918 struct kvm_vcpu *vcpu;
3919 union tod_clock clk;
3924 store_tod_clock_ext(&clk);
3926 kvm->arch.epoch = gtod->tod - clk.tod;
3928 if (test_kvm_facility(kvm, 139)) {
3929 kvm->arch.epdx = gtod->epoch_idx - clk.ei;
3930 if (kvm->arch.epoch > gtod->tod)
3931 kvm->arch.epdx -= 1;
3934 kvm_s390_vcpu_block_all(kvm);
3935 kvm_for_each_vcpu(i, vcpu, kvm) {
3936 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
3937 vcpu->arch.sie_block->epdx = kvm->arch.epdx;
3940 kvm_s390_vcpu_unblock_all(kvm);
3944 void kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
3946 mutex_lock(&kvm->lock);
3947 __kvm_s390_set_tod_clock(kvm, gtod);
3948 mutex_unlock(&kvm->lock);
3951 int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
3953 if (!mutex_trylock(&kvm->lock))
3955 __kvm_s390_set_tod_clock(kvm, gtod);
3956 mutex_unlock(&kvm->lock);
3961 * kvm_arch_fault_in_page - fault-in guest page if necessary
3962 * @vcpu: The corresponding virtual cpu
3963 * @gpa: Guest physical address
3964 * @writable: Whether the page should be writable or not
3966 * Make sure that a guest page has been faulted-in on the host.
3968 * Return: Zero on success, negative error code otherwise.
3970 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
3972 return gmap_fault(vcpu->arch.gmap, gpa,
3973 writable ? FAULT_FLAG_WRITE : 0);
3976 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3977 unsigned long token)
3979 struct kvm_s390_interrupt inti;
3980 struct kvm_s390_irq irq;
3983 irq.u.ext.ext_params2 = token;
3984 irq.type = KVM_S390_INT_PFAULT_INIT;
3985 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3987 inti.type = KVM_S390_INT_PFAULT_DONE;
3988 inti.parm64 = token;
3989 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3993 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3994 struct kvm_async_pf *work)
3996 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3997 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
4002 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
4003 struct kvm_async_pf *work)
4005 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
4006 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
4009 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
4010 struct kvm_async_pf *work)
4012 /* s390 will always inject the page directly */
4015 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
4018 * s390 will always inject the page directly,
4019 * but we still want check_async_completion to cleanup
4024 static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
4027 struct kvm_arch_async_pf arch;
4029 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4031 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
4032 vcpu->arch.pfault_compare)
4034 if (psw_extint_disabled(vcpu))
4036 if (kvm_s390_vcpu_has_irq(vcpu, 0))
4038 if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
4040 if (!vcpu->arch.gmap->pfault_enabled)
4043 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
4044 hva += current->thread.gmap_addr & ~PAGE_MASK;
4045 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
4048 return kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
4051 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
4056 * On s390 notifications for arriving pages will be delivered directly
4057 * to the guest but the house keeping for completed pfaults is
4058 * handled outside the worker.
4060 kvm_check_async_pf_completion(vcpu);
4062 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
4063 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
4068 if (!kvm_is_ucontrol(vcpu->kvm)) {
4069 rc = kvm_s390_deliver_pending_interrupts(vcpu);
4074 rc = kvm_s390_handle_requests(vcpu);
4078 if (guestdbg_enabled(vcpu)) {
4079 kvm_s390_backup_guest_per_regs(vcpu);
4080 kvm_s390_patch_guest_per_regs(vcpu);
4083 clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
4085 vcpu->arch.sie_block->icptcode = 0;
4086 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
4087 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
4088 trace_kvm_s390_sie_enter(vcpu, cpuflags);
4093 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
4095 struct kvm_s390_pgm_info pgm_info = {
4096 .code = PGM_ADDRESSING,
4101 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
4102 trace_kvm_s390_sie_fault(vcpu);
4105 * We want to inject an addressing exception, which is defined as a
4106 * suppressing or terminating exception. However, since we came here
4107 * by a DAT access exception, the PSW still points to the faulting
4108 * instruction since DAT exceptions are nullifying. So we've got
4109 * to look up the current opcode to get the length of the instruction
4110 * to be able to forward the PSW.
4112 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
4113 ilen = insn_length(opcode);
4117 /* Instruction-Fetching Exceptions - we can't detect the ilen.
4118 * Forward by arbitrary ilc, injection will take care of
4119 * nullification if necessary.
4121 pgm_info = vcpu->arch.pgm;
4124 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
4125 kvm_s390_forward_psw(vcpu, ilen);
4126 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
4129 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
4131 struct mcck_volatile_info *mcck_info;
4132 struct sie_page *sie_page;
4134 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
4135 vcpu->arch.sie_block->icptcode);
4136 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
4138 if (guestdbg_enabled(vcpu))
4139 kvm_s390_restore_guest_per_regs(vcpu);
4141 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
4142 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
4144 if (exit_reason == -EINTR) {
4145 VCPU_EVENT(vcpu, 3, "%s", "machine check");
4146 sie_page = container_of(vcpu->arch.sie_block,
4147 struct sie_page, sie_block);
4148 mcck_info = &sie_page->mcck_info;
4149 kvm_s390_reinject_machine_check(vcpu, mcck_info);
4153 if (vcpu->arch.sie_block->icptcode > 0) {
4154 int rc = kvm_handle_sie_intercept(vcpu);
4156 if (rc != -EOPNOTSUPP)
4158 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
4159 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
4160 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
4161 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
4163 } else if (exit_reason != -EFAULT) {
4164 vcpu->stat.exit_null++;
4166 } else if (kvm_is_ucontrol(vcpu->kvm)) {
4167 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
4168 vcpu->run->s390_ucontrol.trans_exc_code =
4169 current->thread.gmap_addr;
4170 vcpu->run->s390_ucontrol.pgm_code = 0x10;
4172 } else if (current->thread.gmap_pfault) {
4173 trace_kvm_s390_major_guest_pfault(vcpu);
4174 current->thread.gmap_pfault = 0;
4175 if (kvm_arch_setup_async_pf(vcpu))
4177 vcpu->stat.pfault_sync++;
4178 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
4180 return vcpu_post_run_fault_in_sie(vcpu);
4183 #define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK)
4184 static int __vcpu_run(struct kvm_vcpu *vcpu)
4186 int rc, exit_reason;
4187 struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block;
4190 * We try to hold kvm->srcu during most of vcpu_run (except when run-
4191 * ning the guest), so that memslots (and other stuff) are protected
4193 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4196 rc = vcpu_pre_run(vcpu);
4200 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
4202 * As PF_VCPU will be used in fault handler, between
4203 * guest_enter and guest_exit should be no uaccess.
4205 local_irq_disable();
4206 guest_enter_irqoff();
4207 __disable_cpu_timer_accounting(vcpu);
4209 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4210 memcpy(sie_page->pv_grregs,
4211 vcpu->run->s.regs.gprs,
4212 sizeof(sie_page->pv_grregs));
4214 if (test_cpu_flag(CIF_FPU))
4216 exit_reason = sie64a(vcpu->arch.sie_block,
4217 vcpu->run->s.regs.gprs);
4218 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4219 memcpy(vcpu->run->s.regs.gprs,
4220 sie_page->pv_grregs,
4221 sizeof(sie_page->pv_grregs));
4223 * We're not allowed to inject interrupts on intercepts
4224 * that leave the guest state in an "in-between" state
4225 * where the next SIE entry will do a continuation.
4226 * Fence interrupts in our "internal" PSW.
4228 if (vcpu->arch.sie_block->icptcode == ICPT_PV_INSTR ||
4229 vcpu->arch.sie_block->icptcode == ICPT_PV_PREF) {
4230 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
4233 local_irq_disable();
4234 __enable_cpu_timer_accounting(vcpu);
4235 guest_exit_irqoff();
4237 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4239 rc = vcpu_post_run(vcpu, exit_reason);
4240 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
4242 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
4246 static void sync_regs_fmt2(struct kvm_vcpu *vcpu)
4248 struct kvm_run *kvm_run = vcpu->run;
4249 struct runtime_instr_cb *riccb;
4252 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
4253 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
4254 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
4255 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
4256 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4257 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
4258 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
4259 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
4261 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
4262 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
4263 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
4264 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
4265 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4266 kvm_clear_async_pf_completion_queue(vcpu);
4268 if (kvm_run->kvm_dirty_regs & KVM_SYNC_DIAG318) {
4269 vcpu->arch.diag318_info.val = kvm_run->s.regs.diag318;
4270 vcpu->arch.sie_block->cpnc = vcpu->arch.diag318_info.cpnc;
4273 * If userspace sets the riccb (e.g. after migration) to a valid state,
4274 * we should enable RI here instead of doing the lazy enablement.
4276 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
4277 test_kvm_facility(vcpu->kvm, 64) &&
4279 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
4280 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
4281 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
4284 * If userspace sets the gscb (e.g. after migration) to non-zero,
4285 * we should enable GS here instead of doing the lazy enablement.
4287 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
4288 test_kvm_facility(vcpu->kvm, 133) &&
4290 !vcpu->arch.gs_enabled) {
4291 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
4292 vcpu->arch.sie_block->ecb |= ECB_GS;
4293 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
4294 vcpu->arch.gs_enabled = 1;
4296 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
4297 test_kvm_facility(vcpu->kvm, 82)) {
4298 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
4299 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
4301 if (MACHINE_HAS_GS) {
4303 __ctl_set_bit(2, 4);
4304 if (current->thread.gs_cb) {
4305 vcpu->arch.host_gscb = current->thread.gs_cb;
4306 save_gs_cb(vcpu->arch.host_gscb);
4308 if (vcpu->arch.gs_enabled) {
4309 current->thread.gs_cb = (struct gs_cb *)
4310 &vcpu->run->s.regs.gscb;
4311 restore_gs_cb(current->thread.gs_cb);
4315 /* SIE will load etoken directly from SDNX and therefore kvm_run */
4318 static void sync_regs(struct kvm_vcpu *vcpu)
4320 struct kvm_run *kvm_run = vcpu->run;
4322 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
4323 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
4324 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
4325 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
4326 /* some control register changes require a tlb flush */
4327 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4329 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4330 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
4331 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
4333 save_access_regs(vcpu->arch.host_acrs);
4334 restore_access_regs(vcpu->run->s.regs.acrs);
4335 /* save host (userspace) fprs/vrs */
4337 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
4338 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
4340 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
4342 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
4343 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
4344 if (test_fp_ctl(current->thread.fpu.fpc))
4345 /* User space provided an invalid FPC, let's clear it */
4346 current->thread.fpu.fpc = 0;
4348 /* Sync fmt2 only data */
4349 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) {
4350 sync_regs_fmt2(vcpu);
4353 * In several places we have to modify our internal view to
4354 * not do things that are disallowed by the ultravisor. For
4355 * example we must not inject interrupts after specific exits
4356 * (e.g. 112 prefix page not secure). We do this by turning
4357 * off the machine check, external and I/O interrupt bits
4358 * of our PSW copy. To avoid getting validity intercepts, we
4359 * do only accept the condition code from userspace.
4361 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_CC;
4362 vcpu->arch.sie_block->gpsw.mask |= kvm_run->psw_mask &
4366 kvm_run->kvm_dirty_regs = 0;
4369 static void store_regs_fmt2(struct kvm_vcpu *vcpu)
4371 struct kvm_run *kvm_run = vcpu->run;
4373 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
4374 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
4375 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
4376 kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
4377 kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val;
4378 if (MACHINE_HAS_GS) {
4380 __ctl_set_bit(2, 4);
4381 if (vcpu->arch.gs_enabled)
4382 save_gs_cb(current->thread.gs_cb);
4383 current->thread.gs_cb = vcpu->arch.host_gscb;
4384 restore_gs_cb(vcpu->arch.host_gscb);
4385 if (!vcpu->arch.host_gscb)
4386 __ctl_clear_bit(2, 4);
4387 vcpu->arch.host_gscb = NULL;
4390 /* SIE will save etoken directly into SDNX and therefore kvm_run */
4393 static void store_regs(struct kvm_vcpu *vcpu)
4395 struct kvm_run *kvm_run = vcpu->run;
4397 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
4398 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
4399 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
4400 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4401 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
4402 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
4403 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
4404 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
4405 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
4406 save_access_regs(vcpu->run->s.regs.acrs);
4407 restore_access_regs(vcpu->arch.host_acrs);
4408 /* Save guest register state */
4410 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
4411 /* Restore will be done lazily at return */
4412 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
4413 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
4414 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu)))
4415 store_regs_fmt2(vcpu);
4418 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
4420 struct kvm_run *kvm_run = vcpu->run;
4423 if (kvm_run->immediate_exit)
4426 if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
4427 kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
4432 if (guestdbg_exit_pending(vcpu)) {
4433 kvm_s390_prepare_debug_exit(vcpu);
4438 kvm_sigset_activate(vcpu);
4441 * no need to check the return value of vcpu_start as it can only have
4442 * an error for protvirt, but protvirt means user cpu state
4444 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
4445 kvm_s390_vcpu_start(vcpu);
4446 } else if (is_vcpu_stopped(vcpu)) {
4447 pr_err_ratelimited("can't run stopped vcpu %d\n",
4454 enable_cpu_timer_accounting(vcpu);
4457 rc = __vcpu_run(vcpu);
4459 if (signal_pending(current) && !rc) {
4460 kvm_run->exit_reason = KVM_EXIT_INTR;
4464 if (guestdbg_exit_pending(vcpu) && !rc) {
4465 kvm_s390_prepare_debug_exit(vcpu);
4469 if (rc == -EREMOTE) {
4470 /* userspace support is needed, kvm_run has been prepared */
4474 disable_cpu_timer_accounting(vcpu);
4477 kvm_sigset_deactivate(vcpu);
4479 vcpu->stat.exit_userspace++;
4486 * store status at address
4487 * we use have two special cases:
4488 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
4489 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
4491 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
4493 unsigned char archmode = 1;
4494 freg_t fprs[NUM_FPRS];
4499 px = kvm_s390_get_prefix(vcpu);
4500 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
4501 if (write_guest_abs(vcpu, 163, &archmode, 1))
4504 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
4505 if (write_guest_real(vcpu, 163, &archmode, 1))
4509 gpa -= __LC_FPREGS_SAVE_AREA;
4511 /* manually convert vector registers if necessary */
4512 if (MACHINE_HAS_VX) {
4513 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
4514 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
4517 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
4518 vcpu->run->s.regs.fprs, 128);
4520 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
4521 vcpu->run->s.regs.gprs, 128);
4522 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
4523 &vcpu->arch.sie_block->gpsw, 16);
4524 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
4526 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
4527 &vcpu->run->s.regs.fpc, 4);
4528 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
4529 &vcpu->arch.sie_block->todpr, 4);
4530 cputm = kvm_s390_get_cpu_timer(vcpu);
4531 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4533 clkcomp = vcpu->arch.sie_block->ckc >> 8;
4534 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
4536 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
4537 &vcpu->run->s.regs.acrs, 64);
4538 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
4539 &vcpu->arch.sie_block->gcr, 128);
4540 return rc ? -EFAULT : 0;
4543 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
4546 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
4547 * switch in the run ioctl. Let's update our copies before we save
4548 * it into the save area
4551 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
4552 save_access_regs(vcpu->run->s.regs.acrs);
4554 return kvm_s390_store_status_unloaded(vcpu, addr);
4557 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4559 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
4560 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
4563 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
4566 struct kvm_vcpu *vcpu;
4568 kvm_for_each_vcpu(i, vcpu, kvm) {
4569 __disable_ibs_on_vcpu(vcpu);
4573 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4577 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
4578 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
4581 int kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
4583 int i, online_vcpus, r = 0, started_vcpus = 0;
4585 if (!is_vcpu_stopped(vcpu))
4588 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
4589 /* Only one cpu at a time may enter/leave the STOPPED state. */
4590 spin_lock(&vcpu->kvm->arch.start_stop_lock);
4591 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4593 /* Let's tell the UV that we want to change into the operating state */
4594 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4595 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR);
4597 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4602 for (i = 0; i < online_vcpus; i++) {
4603 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
4607 if (started_vcpus == 0) {
4608 /* we're the only active VCPU -> speed it up */
4609 __enable_ibs_on_vcpu(vcpu);
4610 } else if (started_vcpus == 1) {
4612 * As we are starting a second VCPU, we have to disable
4613 * the IBS facility on all VCPUs to remove potentially
4614 * outstanding ENABLE requests.
4616 __disable_ibs_on_all_vcpus(vcpu->kvm);
4619 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
4621 * The real PSW might have changed due to a RESTART interpreted by the
4622 * ultravisor. We block all interrupts and let the next sie exit
4625 if (kvm_s390_pv_cpu_is_protected(vcpu))
4626 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
4628 * Another VCPU might have used IBS while we were offline.
4629 * Let's play safe and flush the VCPU at startup.
4631 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4632 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4636 int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
4638 int i, online_vcpus, r = 0, started_vcpus = 0;
4639 struct kvm_vcpu *started_vcpu = NULL;
4641 if (is_vcpu_stopped(vcpu))
4644 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
4645 /* Only one cpu at a time may enter/leave the STOPPED state. */
4646 spin_lock(&vcpu->kvm->arch.start_stop_lock);
4647 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4649 /* Let's tell the UV that we want to change into the stopped state */
4650 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4651 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_STP);
4653 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4659 * Set the VCPU to STOPPED and THEN clear the interrupt flag,
4660 * now that the SIGP STOP and SIGP STOP AND STORE STATUS orders
4661 * have been fully processed. This will ensure that the VCPU
4662 * is kept BUSY if another VCPU is inquiring with SIGP SENSE.
4664 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
4665 kvm_s390_clear_stop_irq(vcpu);
4667 __disable_ibs_on_vcpu(vcpu);
4669 for (i = 0; i < online_vcpus; i++) {
4670 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
4672 started_vcpu = vcpu->kvm->vcpus[i];
4676 if (started_vcpus == 1) {
4678 * As we only have one VCPU left, we want to enable the
4679 * IBS facility for that VCPU to speed it up.
4681 __enable_ibs_on_vcpu(started_vcpu);
4684 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4688 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
4689 struct kvm_enable_cap *cap)
4697 case KVM_CAP_S390_CSS_SUPPORT:
4698 if (!vcpu->kvm->arch.css_support) {
4699 vcpu->kvm->arch.css_support = 1;
4700 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
4701 trace_kvm_s390_enable_css(vcpu->kvm);
4712 static long kvm_s390_guest_sida_op(struct kvm_vcpu *vcpu,
4713 struct kvm_s390_mem_op *mop)
4715 void __user *uaddr = (void __user *)mop->buf;
4718 if (mop->flags || !mop->size)
4720 if (mop->size + mop->sida_offset < mop->size)
4722 if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block))
4724 if (!kvm_s390_pv_cpu_is_protected(vcpu))
4728 case KVM_S390_MEMOP_SIDA_READ:
4729 if (copy_to_user(uaddr, (void *)(sida_origin(vcpu->arch.sie_block) +
4730 mop->sida_offset), mop->size))
4734 case KVM_S390_MEMOP_SIDA_WRITE:
4735 if (copy_from_user((void *)(sida_origin(vcpu->arch.sie_block) +
4736 mop->sida_offset), uaddr, mop->size))
4742 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
4743 struct kvm_s390_mem_op *mop)
4745 void __user *uaddr = (void __user *)mop->buf;
4746 void *tmpbuf = NULL;
4748 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
4749 | KVM_S390_MEMOP_F_CHECK_ONLY;
4751 if (mop->flags & ~supported_flags || mop->ar >= NUM_ACRS || !mop->size)
4754 if (mop->size > MEM_OP_MAX_SIZE)
4757 if (kvm_s390_pv_cpu_is_protected(vcpu))
4760 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
4761 tmpbuf = vmalloc(mop->size);
4767 case KVM_S390_MEMOP_LOGICAL_READ:
4768 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4769 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4770 mop->size, GACC_FETCH);
4773 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4775 if (copy_to_user(uaddr, tmpbuf, mop->size))
4779 case KVM_S390_MEMOP_LOGICAL_WRITE:
4780 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4781 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4782 mop->size, GACC_STORE);
4785 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
4789 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4793 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
4794 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
4800 static long kvm_s390_guest_memsida_op(struct kvm_vcpu *vcpu,
4801 struct kvm_s390_mem_op *mop)
4805 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4808 case KVM_S390_MEMOP_LOGICAL_READ:
4809 case KVM_S390_MEMOP_LOGICAL_WRITE:
4810 r = kvm_s390_guest_mem_op(vcpu, mop);
4812 case KVM_S390_MEMOP_SIDA_READ:
4813 case KVM_S390_MEMOP_SIDA_WRITE:
4814 /* we are locked against sida going away by the vcpu->mutex */
4815 r = kvm_s390_guest_sida_op(vcpu, mop);
4821 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
4825 long kvm_arch_vcpu_async_ioctl(struct file *filp,
4826 unsigned int ioctl, unsigned long arg)
4828 struct kvm_vcpu *vcpu = filp->private_data;
4829 void __user *argp = (void __user *)arg;
4832 case KVM_S390_IRQ: {
4833 struct kvm_s390_irq s390irq;
4835 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
4837 return kvm_s390_inject_vcpu(vcpu, &s390irq);
4839 case KVM_S390_INTERRUPT: {
4840 struct kvm_s390_interrupt s390int;
4841 struct kvm_s390_irq s390irq = {};
4843 if (copy_from_user(&s390int, argp, sizeof(s390int)))
4845 if (s390int_to_s390irq(&s390int, &s390irq))
4847 return kvm_s390_inject_vcpu(vcpu, &s390irq);
4850 return -ENOIOCTLCMD;
4853 long kvm_arch_vcpu_ioctl(struct file *filp,
4854 unsigned int ioctl, unsigned long arg)
4856 struct kvm_vcpu *vcpu = filp->private_data;
4857 void __user *argp = (void __user *)arg;
4865 case KVM_S390_STORE_STATUS:
4866 idx = srcu_read_lock(&vcpu->kvm->srcu);
4867 r = kvm_s390_store_status_unloaded(vcpu, arg);
4868 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4870 case KVM_S390_SET_INITIAL_PSW: {
4874 if (copy_from_user(&psw, argp, sizeof(psw)))
4876 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
4879 case KVM_S390_CLEAR_RESET:
4881 kvm_arch_vcpu_ioctl_clear_reset(vcpu);
4882 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4883 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
4884 UVC_CMD_CPU_RESET_CLEAR, &rc, &rrc);
4885 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET CLEAR VCPU: rc %x rrc %x",
4889 case KVM_S390_INITIAL_RESET:
4891 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4892 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4893 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
4894 UVC_CMD_CPU_RESET_INITIAL,
4896 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET INITIAL VCPU: rc %x rrc %x",
4900 case KVM_S390_NORMAL_RESET:
4902 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
4903 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4904 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
4905 UVC_CMD_CPU_RESET, &rc, &rrc);
4906 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: rc %x rrc %x",
4910 case KVM_SET_ONE_REG:
4911 case KVM_GET_ONE_REG: {
4912 struct kvm_one_reg reg;
4914 if (kvm_s390_pv_cpu_is_protected(vcpu))
4917 if (copy_from_user(®, argp, sizeof(reg)))
4919 if (ioctl == KVM_SET_ONE_REG)
4920 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®);
4922 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®);
4925 #ifdef CONFIG_KVM_S390_UCONTROL
4926 case KVM_S390_UCAS_MAP: {
4927 struct kvm_s390_ucas_mapping ucasmap;
4929 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4934 if (!kvm_is_ucontrol(vcpu->kvm)) {
4939 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
4940 ucasmap.vcpu_addr, ucasmap.length);
4943 case KVM_S390_UCAS_UNMAP: {
4944 struct kvm_s390_ucas_mapping ucasmap;
4946 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4951 if (!kvm_is_ucontrol(vcpu->kvm)) {
4956 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
4961 case KVM_S390_VCPU_FAULT: {
4962 r = gmap_fault(vcpu->arch.gmap, arg, 0);
4965 case KVM_ENABLE_CAP:
4967 struct kvm_enable_cap cap;
4969 if (copy_from_user(&cap, argp, sizeof(cap)))
4971 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
4974 case KVM_S390_MEM_OP: {
4975 struct kvm_s390_mem_op mem_op;
4977 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
4978 r = kvm_s390_guest_memsida_op(vcpu, &mem_op);
4983 case KVM_S390_SET_IRQ_STATE: {
4984 struct kvm_s390_irq_state irq_state;
4987 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4989 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
4990 irq_state.len == 0 ||
4991 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
4995 /* do not use irq_state.flags, it will break old QEMUs */
4996 r = kvm_s390_set_irq_state(vcpu,
4997 (void __user *) irq_state.buf,
5001 case KVM_S390_GET_IRQ_STATE: {
5002 struct kvm_s390_irq_state irq_state;
5005 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
5007 if (irq_state.len == 0) {
5011 /* do not use irq_state.flags, it will break old QEMUs */
5012 r = kvm_s390_get_irq_state(vcpu,
5013 (__u8 __user *) irq_state.buf,
5025 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
5027 #ifdef CONFIG_KVM_S390_UCONTROL
5028 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
5029 && (kvm_is_ucontrol(vcpu->kvm))) {
5030 vmf->page = virt_to_page(vcpu->arch.sie_block);
5031 get_page(vmf->page);
5035 return VM_FAULT_SIGBUS;
5038 /* Section: memory related */
5039 int kvm_arch_prepare_memory_region(struct kvm *kvm,
5040 struct kvm_memory_slot *memslot,
5041 const struct kvm_userspace_memory_region *mem,
5042 enum kvm_mr_change change)
5044 /* A few sanity checks. We can have memory slots which have to be
5045 located/ended at a segment boundary (1MB). The memory in userland is
5046 ok to be fragmented into various different vmas. It is okay to mmap()
5047 and munmap() stuff in this slot after doing this call at any time */
5049 if (mem->userspace_addr & 0xffffful)
5052 if (mem->memory_size & 0xffffful)
5055 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
5058 /* When we are protected, we should not change the memory slots */
5059 if (kvm_s390_pv_get_handle(kvm))
5064 void kvm_arch_commit_memory_region(struct kvm *kvm,
5065 const struct kvm_userspace_memory_region *mem,
5066 struct kvm_memory_slot *old,
5067 const struct kvm_memory_slot *new,
5068 enum kvm_mr_change change)
5074 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
5075 old->npages * PAGE_SIZE);
5078 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
5079 old->npages * PAGE_SIZE);
5084 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
5085 mem->guest_phys_addr, mem->memory_size);
5087 case KVM_MR_FLAGS_ONLY:
5090 WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
5093 pr_warn("failed to commit memory region\n");
5097 static inline unsigned long nonhyp_mask(int i)
5099 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
5101 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
5104 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
5106 vcpu->valid_wakeup = false;
5109 static int __init kvm_s390_init(void)
5113 if (!sclp.has_sief2) {
5114 pr_info("SIE is not available\n");
5118 if (nested && hpage) {
5119 pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n");
5123 for (i = 0; i < 16; i++)
5124 kvm_s390_fac_base[i] |=
5125 stfle_fac_list[i] & nonhyp_mask(i);
5127 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
5130 static void __exit kvm_s390_exit(void)
5135 module_init(kvm_s390_init);
5136 module_exit(kvm_s390_exit);
5139 * Enable autoloading of the kvm module.
5140 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
5141 * since x86 takes a different approach.
5143 #include <linux/miscdevice.h>
5144 MODULE_ALIAS_MISCDEV(KVM_MINOR);
5145 MODULE_ALIAS("devname:kvm");