4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "qemu/osdep.h"
20 #include "qapi/error.h"
24 #include "qemu/cutils.h"
26 #include "exec/exec-all.h"
28 #include "hw/qdev-core.h"
29 #if !defined(CONFIG_USER_ONLY)
30 #include "hw/boards.h"
31 #include "hw/xen/xen.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/hax.h"
35 #include "sysemu/sysemu.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "qemu/error-report.h"
39 #if defined(CONFIG_USER_ONLY)
41 #else /* !CONFIG_USER_ONLY */
43 #include "exec/memory.h"
44 #include "exec/ioport.h"
45 #include "sysemu/dma.h"
46 #include "exec/address-spaces.h"
47 #include "sysemu/xen-mapcache.h"
50 #include "exec/cpu-all.h"
51 #include "qemu/rcu_queue.h"
52 #include "qemu/main-loop.h"
53 #include "translate-all.h"
54 #include "sysemu/replay.h"
56 #include "exec/memory-internal.h"
57 #include "exec/ram_addr.h"
60 #include "migration/vmstate.h"
62 #include "qemu/range.h"
64 #include "qemu/mmap-alloc.h"
67 //#define DEBUG_SUBPAGE
69 #if !defined(CONFIG_USER_ONLY)
70 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
71 * are protected by the ramlist lock.
73 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
75 static MemoryRegion *system_memory;
76 static MemoryRegion *system_io;
78 AddressSpace address_space_io;
79 AddressSpace address_space_memory;
81 MemoryRegion io_mem_rom, io_mem_notdirty;
82 static MemoryRegion io_mem_unassigned;
84 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
85 #define RAM_PREALLOC (1 << 0)
87 /* RAM is mmap-ed with MAP_SHARED */
88 #define RAM_SHARED (1 << 1)
90 /* Only a portion of RAM (used_length) is actually used, and migrated.
91 * This used_length size can change across reboots.
93 #define RAM_RESIZEABLE (1 << 2)
97 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
98 /* current CPU in the current thread. It is only valid inside
100 __thread CPUState *current_cpu;
101 /* 0 = Do not count executed instructions.
102 1 = Precise instruction counting.
103 2 = Adaptive rate instruction counting. */
106 #if !defined(CONFIG_USER_ONLY)
108 typedef struct PhysPageEntry PhysPageEntry;
110 struct PhysPageEntry {
111 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
113 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
117 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
119 /* Size of the L2 (and L3, etc) page tables. */
120 #define ADDR_SPACE_BITS 64
123 #define P_L2_SIZE (1 << P_L2_BITS)
125 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
127 typedef PhysPageEntry Node[P_L2_SIZE];
129 typedef struct PhysPageMap {
132 unsigned sections_nb;
133 unsigned sections_nb_alloc;
135 unsigned nodes_nb_alloc;
137 MemoryRegionSection *sections;
140 struct AddressSpaceDispatch {
143 MemoryRegionSection *mru_section;
144 /* This is a multi-level map on the physical address space.
145 * The bottom level has pointers to MemoryRegionSections.
147 PhysPageEntry phys_map;
152 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
153 typedef struct subpage_t {
157 uint16_t sub_section[TARGET_PAGE_SIZE];
160 #define PHYS_SECTION_UNASSIGNED 0
161 #define PHYS_SECTION_NOTDIRTY 1
162 #define PHYS_SECTION_ROM 2
163 #define PHYS_SECTION_WATCH 3
165 static void io_mem_init(void);
166 static void memory_map_init(void);
167 static void tcg_commit(MemoryListener *listener);
169 static MemoryRegion io_mem_watch;
172 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
173 * @cpu: the CPU whose AddressSpace this is
174 * @as: the AddressSpace itself
175 * @memory_dispatch: its dispatch pointer (cached, RCU protected)
176 * @tcg_as_listener: listener for tracking changes to the AddressSpace
178 struct CPUAddressSpace {
181 struct AddressSpaceDispatch *memory_dispatch;
182 MemoryListener tcg_as_listener;
187 #if !defined(CONFIG_USER_ONLY)
189 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
191 static unsigned alloc_hint = 16;
192 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
193 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, alloc_hint);
194 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
195 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
196 alloc_hint = map->nodes_nb_alloc;
200 static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
207 ret = map->nodes_nb++;
209 assert(ret != PHYS_MAP_NODE_NIL);
210 assert(ret != map->nodes_nb_alloc);
212 e.skip = leaf ? 0 : 1;
213 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
214 for (i = 0; i < P_L2_SIZE; ++i) {
215 memcpy(&p[i], &e, sizeof(e));
220 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
221 hwaddr *index, hwaddr *nb, uint16_t leaf,
225 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
227 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
228 lp->ptr = phys_map_node_alloc(map, level == 0);
230 p = map->nodes[lp->ptr];
231 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
233 while (*nb && lp < &p[P_L2_SIZE]) {
234 if ((*index & (step - 1)) == 0 && *nb >= step) {
240 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
246 static void phys_page_set(AddressSpaceDispatch *d,
247 hwaddr index, hwaddr nb,
250 /* Wildly overreserve - it doesn't matter much. */
251 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
253 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
256 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
257 * and update our entry so we can skip it and go directly to the destination.
259 static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
261 unsigned valid_ptr = P_L2_SIZE;
266 if (lp->ptr == PHYS_MAP_NODE_NIL) {
271 for (i = 0; i < P_L2_SIZE; i++) {
272 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
279 phys_page_compact(&p[i], nodes, compacted);
283 /* We can only compress if there's only one child. */
288 assert(valid_ptr < P_L2_SIZE);
290 /* Don't compress if it won't fit in the # of bits we have. */
291 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
295 lp->ptr = p[valid_ptr].ptr;
296 if (!p[valid_ptr].skip) {
297 /* If our only child is a leaf, make this a leaf. */
298 /* By design, we should have made this node a leaf to begin with so we
299 * should never reach here.
300 * But since it's so simple to handle this, let's do it just in case we
305 lp->skip += p[valid_ptr].skip;
309 static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
311 DECLARE_BITMAP(compacted, nodes_nb);
313 if (d->phys_map.skip) {
314 phys_page_compact(&d->phys_map, d->map.nodes, compacted);
318 static inline bool section_covers_addr(const MemoryRegionSection *section,
321 /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
322 * the section must cover the entire address space.
324 return section->size.hi ||
325 range_covers_byte(section->offset_within_address_space,
326 section->size.lo, addr);
329 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
330 Node *nodes, MemoryRegionSection *sections)
333 hwaddr index = addr >> TARGET_PAGE_BITS;
336 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
337 if (lp.ptr == PHYS_MAP_NODE_NIL) {
338 return §ions[PHYS_SECTION_UNASSIGNED];
341 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
344 if (section_covers_addr(§ions[lp.ptr], addr)) {
345 return §ions[lp.ptr];
347 return §ions[PHYS_SECTION_UNASSIGNED];
351 bool memory_region_is_unassigned(MemoryRegion *mr)
353 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
354 && mr != &io_mem_watch;
357 /* Called from RCU critical section */
358 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
360 bool resolve_subpage)
362 MemoryRegionSection *section = atomic_read(&d->mru_section);
366 if (section && section != &d->map.sections[PHYS_SECTION_UNASSIGNED] &&
367 section_covers_addr(section, addr)) {
370 section = phys_page_find(d->phys_map, addr, d->map.nodes,
374 if (resolve_subpage && section->mr->subpage) {
375 subpage = container_of(section->mr, subpage_t, iomem);
376 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
379 atomic_set(&d->mru_section, section);
384 /* Called from RCU critical section */
385 static MemoryRegionSection *
386 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
387 hwaddr *plen, bool resolve_subpage)
389 MemoryRegionSection *section;
393 section = address_space_lookup_region(d, addr, resolve_subpage);
394 /* Compute offset within MemoryRegionSection */
395 addr -= section->offset_within_address_space;
397 /* Compute offset within MemoryRegion */
398 *xlat = addr + section->offset_within_region;
402 /* MMIO registers can be expected to perform full-width accesses based only
403 * on their address, without considering adjacent registers that could
404 * decode to completely different MemoryRegions. When such registers
405 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
406 * regions overlap wildly. For this reason we cannot clamp the accesses
409 * If the length is small (as is the case for address_space_ldl/stl),
410 * everything works fine. If the incoming length is large, however,
411 * the caller really has to do the clamping through memory_access_size.
413 if (memory_region_is_ram(mr)) {
414 diff = int128_sub(section->size, int128_make64(addr));
415 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
420 /* Called from RCU critical section */
421 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
422 hwaddr *xlat, hwaddr *plen,
426 MemoryRegionSection *section;
430 AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
431 section = address_space_translate_internal(d, addr, &addr, plen, true);
434 if (!mr->iommu_ops) {
438 iotlb = mr->iommu_ops->translate(mr, addr, is_write);
439 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
440 | (addr & iotlb.addr_mask));
441 *plen = MIN(*plen, (addr | iotlb.addr_mask) - addr + 1);
442 if (!(iotlb.perm & (1 << is_write))) {
443 mr = &io_mem_unassigned;
447 as = iotlb.target_as;
450 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
451 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
452 *plen = MIN(page, *plen);
459 /* Called from RCU critical section */
460 MemoryRegionSection *
461 address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
462 hwaddr *xlat, hwaddr *plen)
464 MemoryRegionSection *section;
465 AddressSpaceDispatch *d = cpu->cpu_ases[asidx].memory_dispatch;
467 section = address_space_translate_internal(d, addr, xlat, plen, false);
469 assert(!section->mr->iommu_ops);
474 #if !defined(CONFIG_USER_ONLY)
476 static int cpu_common_post_load(void *opaque, int version_id)
478 CPUState *cpu = opaque;
480 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
481 version_id is increased. */
482 cpu->interrupt_request &= ~0x01;
488 static int cpu_common_pre_load(void *opaque)
490 CPUState *cpu = opaque;
492 cpu->exception_index = -1;
497 static bool cpu_common_exception_index_needed(void *opaque)
499 CPUState *cpu = opaque;
501 return tcg_enabled() && cpu->exception_index != -1;
504 static const VMStateDescription vmstate_cpu_common_exception_index = {
505 .name = "cpu_common/exception_index",
507 .minimum_version_id = 1,
508 .needed = cpu_common_exception_index_needed,
509 .fields = (VMStateField[]) {
510 VMSTATE_INT32(exception_index, CPUState),
511 VMSTATE_END_OF_LIST()
515 static bool cpu_common_crash_occurred_needed(void *opaque)
517 CPUState *cpu = opaque;
519 return cpu->crash_occurred;
522 static const VMStateDescription vmstate_cpu_common_crash_occurred = {
523 .name = "cpu_common/crash_occurred",
525 .minimum_version_id = 1,
526 .needed = cpu_common_crash_occurred_needed,
527 .fields = (VMStateField[]) {
528 VMSTATE_BOOL(crash_occurred, CPUState),
529 VMSTATE_END_OF_LIST()
533 const VMStateDescription vmstate_cpu_common = {
534 .name = "cpu_common",
536 .minimum_version_id = 1,
537 .pre_load = cpu_common_pre_load,
538 .post_load = cpu_common_post_load,
539 .fields = (VMStateField[]) {
540 VMSTATE_UINT32(halted, CPUState),
541 VMSTATE_UINT32(interrupt_request, CPUState),
542 VMSTATE_END_OF_LIST()
544 .subsections = (const VMStateDescription*[]) {
545 &vmstate_cpu_common_exception_index,
546 &vmstate_cpu_common_crash_occurred,
553 CPUState *qemu_get_cpu(int index)
558 if (cpu->cpu_index == index) {
566 #if !defined(CONFIG_USER_ONLY)
567 void cpu_address_space_init(CPUState *cpu, AddressSpace *as, int asidx)
569 CPUAddressSpace *newas;
571 /* Target code should have set num_ases before calling us */
572 assert(asidx < cpu->num_ases);
575 /* address space 0 gets the convenience alias */
579 /* KVM cannot currently support multiple address spaces. */
580 assert(asidx == 0 || !kvm_enabled());
582 if (!cpu->cpu_ases) {
583 cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
586 newas = &cpu->cpu_ases[asidx];
590 newas->tcg_as_listener.commit = tcg_commit;
591 memory_listener_register(&newas->tcg_as_listener, as);
595 AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
597 /* Return the AddressSpace corresponding to the specified index */
598 return cpu->cpu_ases[asidx].as;
602 static int cpu_get_free_index(void)
607 CPU_FOREACH(some_cpu) {
613 void cpu_exec_exit(CPUState *cpu)
615 CPUClass *cc = CPU_GET_CLASS(cpu);
618 if (cpu->node.tqe_prev == NULL) {
619 /* there is nothing to undo since cpu_exec_init() hasn't been called */
624 QTAILQ_REMOVE(&cpus, cpu, node);
625 cpu->node.tqe_prev = NULL;
626 cpu->cpu_index = UNASSIGNED_CPU_INDEX;
629 if (cc->vmsd != NULL) {
630 vmstate_unregister(NULL, cc->vmsd, cpu);
632 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
633 vmstate_unregister(NULL, &vmstate_cpu_common, cpu);
637 void cpu_exec_init(CPUState *cpu, Error **errp)
639 CPUClass *cc ATTRIBUTE_UNUSED = CPU_GET_CLASS(cpu);
640 Error *local_err ATTRIBUTE_UNUSED = NULL;
645 #ifndef CONFIG_USER_ONLY
646 cpu->thread_id = qemu_get_thread_id();
648 /* This is a softmmu CPU object, so create a property for it
649 * so users can wire up its memory. (This can't go in qom/cpu.c
650 * because that file is compiled only once for both user-mode
651 * and system builds.) The default if no link is set up is to use
652 * the system address space.
654 object_property_add_link(OBJECT(cpu), "memory", TYPE_MEMORY_REGION,
655 (Object **)&cpu->memory,
656 qdev_prop_allow_set_link_before_realize,
657 OBJ_PROP_LINK_UNREF_ON_RELEASE,
659 cpu->memory = system_memory;
660 object_ref(OBJECT(cpu->memory));
664 if (cpu->cpu_index == UNASSIGNED_CPU_INDEX) {
665 cpu->cpu_index = cpu_get_free_index();
666 assert(cpu->cpu_index != UNASSIGNED_CPU_INDEX);
668 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
671 #ifndef CONFIG_USER_ONLY
672 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
673 vmstate_register(NULL, cpu->cpu_index, &vmstate_cpu_common, cpu);
675 if (cc->vmsd != NULL) {
676 vmstate_register(NULL, cpu->cpu_index, cc->vmsd, cpu);
681 #if defined(CONFIG_USER_ONLY)
682 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
684 tb_invalidate_phys_page_range(pc, pc + 1, 0);
687 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
690 hwaddr phys = cpu_get_phys_page_attrs_debug(cpu, pc, &attrs);
691 int asidx = cpu_asidx_from_attrs(cpu, attrs);
693 tb_invalidate_phys_addr(cpu->cpu_ases[asidx].as,
694 phys | (pc & ~TARGET_PAGE_MASK));
699 #if defined(CONFIG_USER_ONLY)
700 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
705 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
711 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
715 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
716 int flags, CPUWatchpoint **watchpoint)
721 /* Add a watchpoint. */
722 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
723 int flags, CPUWatchpoint **watchpoint)
727 /* forbid ranges which are empty or run off the end of the address space */
728 if (len == 0 || (addr + len - 1) < addr) {
729 error_report("tried to set invalid watchpoint at %"
730 VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
733 wp = g_malloc(sizeof(*wp));
739 /* keep all GDB-injected watchpoints in front */
740 if (flags & BP_GDB) {
741 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
743 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
746 tlb_flush_page(cpu, addr);
753 /* Remove a specific watchpoint. */
754 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
759 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
760 if (addr == wp->vaddr && len == wp->len
761 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
762 cpu_watchpoint_remove_by_ref(cpu, wp);
769 /* Remove a specific watchpoint by reference. */
770 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
772 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
774 tlb_flush_page(cpu, watchpoint->vaddr);
779 /* Remove all matching watchpoints. */
780 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
782 CPUWatchpoint *wp, *next;
784 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
785 if (wp->flags & mask) {
786 cpu_watchpoint_remove_by_ref(cpu, wp);
791 /* Return true if this watchpoint address matches the specified
792 * access (ie the address range covered by the watchpoint overlaps
793 * partially or completely with the address range covered by the
796 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
800 /* We know the lengths are non-zero, but a little caution is
801 * required to avoid errors in the case where the range ends
802 * exactly at the top of the address space and so addr + len
803 * wraps round to zero.
805 vaddr wpend = wp->vaddr + wp->len - 1;
806 vaddr addrend = addr + len - 1;
808 return !(addr > wpend || wp->vaddr > addrend);
813 /* Add a breakpoint. */
814 int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
815 CPUBreakpoint **breakpoint)
819 bp = g_malloc(sizeof(*bp));
824 /* keep all GDB-injected breakpoints in front */
825 if (flags & BP_GDB) {
826 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
828 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
831 breakpoint_invalidate(cpu, pc);
839 /* Remove a specific breakpoint. */
840 int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
844 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
845 if (bp->pc == pc && bp->flags == flags) {
846 cpu_breakpoint_remove_by_ref(cpu, bp);
853 /* Remove a specific breakpoint by reference. */
854 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
856 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
858 breakpoint_invalidate(cpu, breakpoint->pc);
863 /* Remove all matching breakpoints. */
864 void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
866 CPUBreakpoint *bp, *next;
868 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
869 if (bp->flags & mask) {
870 cpu_breakpoint_remove_by_ref(cpu, bp);
875 /* enable or disable single step mode. EXCP_DEBUG is returned by the
876 CPU loop after each instruction */
877 void cpu_single_step(CPUState *cpu, int enabled)
879 if (cpu->singlestep_enabled != enabled) {
880 cpu->singlestep_enabled = enabled;
882 kvm_update_guest_debug(cpu, 0);
884 /* must flush all the translated code to avoid inconsistencies */
885 /* XXX: only flush what is necessary */
891 void cpu_abort(CPUState *cpu, const char *fmt, ...)
898 fprintf(stderr, "qemu: fatal: ");
899 vfprintf(stderr, fmt, ap);
900 fprintf(stderr, "\n");
901 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
902 if (qemu_log_separate()) {
903 qemu_log("qemu: fatal: ");
904 qemu_log_vprintf(fmt, ap2);
906 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
913 #if defined(CONFIG_USER_ONLY)
915 struct sigaction act;
916 sigfillset(&act.sa_mask);
917 act.sa_handler = SIG_DFL;
918 sigaction(SIGABRT, &act, NULL);
924 #if !defined(CONFIG_USER_ONLY)
925 /* Called from RCU critical section */
926 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
930 block = atomic_rcu_read(&ram_list.mru_block);
931 if (block && addr - block->offset < block->max_length) {
934 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
935 if (addr - block->offset < block->max_length) {
940 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
944 /* It is safe to write mru_block outside the iothread lock. This
949 * xxx removed from list
953 * call_rcu(reclaim_ramblock, xxx);
956 * atomic_rcu_set is not needed here. The block was already published
957 * when it was placed into the list. Here we're just making an extra
958 * copy of the pointer.
960 ram_list.mru_block = block;
964 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
971 end = TARGET_PAGE_ALIGN(start + length);
972 start &= TARGET_PAGE_MASK;
975 block = qemu_get_ram_block(start);
976 assert(block == qemu_get_ram_block(end - 1));
977 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
979 tlb_reset_dirty(cpu, start1, length);
984 /* Note: start and end must be within the same ram block. */
985 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
989 DirtyMemoryBlocks *blocks;
990 unsigned long end, page;
997 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
998 page = start >> TARGET_PAGE_BITS;
1002 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
1004 while (page < end) {
1005 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
1006 unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
1007 unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
1009 dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
1016 if (dirty && tcg_enabled()) {
1017 tlb_reset_dirty_range_all(start, length);
1023 /* Called from RCU critical section */
1024 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
1025 MemoryRegionSection *section,
1027 hwaddr paddr, hwaddr xlat,
1029 target_ulong *address)
1034 if (memory_region_is_ram(section->mr)) {
1036 iotlb = memory_region_get_ram_addr(section->mr) + xlat;
1037 if (!section->readonly) {
1038 iotlb |= PHYS_SECTION_NOTDIRTY;
1040 iotlb |= PHYS_SECTION_ROM;
1043 AddressSpaceDispatch *d;
1045 d = atomic_rcu_read(§ion->address_space->dispatch);
1046 iotlb = section - d->map.sections;
1050 /* Make accesses to pages with watchpoints go via the
1051 watchpoint trap routines. */
1052 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
1053 if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
1054 /* Avoid trapping reads of pages with a write breakpoint. */
1055 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
1056 iotlb = PHYS_SECTION_WATCH + paddr;
1057 *address |= TLB_MMIO;
1065 #endif /* defined(CONFIG_USER_ONLY) */
1067 #if !defined(CONFIG_USER_ONLY)
1069 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1071 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
1073 static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
1074 qemu_anon_ram_alloc;
1077 * Set a custom physical guest memory alloator.
1078 * Accelerators with unusual needs may need this. Hopefully, we can
1079 * get rid of it eventually.
1081 void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))
1083 phys_mem_alloc = alloc;
1086 static uint16_t phys_section_add(PhysPageMap *map,
1087 MemoryRegionSection *section)
1089 /* The physical section number is ORed with a page-aligned
1090 * pointer to produce the iotlb entries. Thus it should
1091 * never overflow into the page-aligned value.
1093 assert(map->sections_nb < TARGET_PAGE_SIZE);
1095 if (map->sections_nb == map->sections_nb_alloc) {
1096 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1097 map->sections = g_renew(MemoryRegionSection, map->sections,
1098 map->sections_nb_alloc);
1100 map->sections[map->sections_nb] = *section;
1101 memory_region_ref(section->mr);
1102 return map->sections_nb++;
1105 static void phys_section_destroy(MemoryRegion *mr)
1107 bool have_sub_page = mr->subpage;
1109 memory_region_unref(mr);
1111 if (have_sub_page) {
1112 subpage_t *subpage = container_of(mr, subpage_t, iomem);
1113 object_unref(OBJECT(&subpage->iomem));
1118 static void phys_sections_free(PhysPageMap *map)
1120 while (map->sections_nb > 0) {
1121 MemoryRegionSection *section = &map->sections[--map->sections_nb];
1122 phys_section_destroy(section->mr);
1124 g_free(map->sections);
1128 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
1131 hwaddr base = section->offset_within_address_space
1133 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
1134 d->map.nodes, d->map.sections);
1135 MemoryRegionSection subsection = {
1136 .offset_within_address_space = base,
1137 .size = int128_make64(TARGET_PAGE_SIZE),
1141 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
1143 if (!(existing->mr->subpage)) {
1144 subpage = subpage_init(d->as, base);
1145 subsection.address_space = d->as;
1146 subsection.mr = &subpage->iomem;
1147 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
1148 phys_section_add(&d->map, &subsection));
1150 subpage = container_of(existing->mr, subpage_t, iomem);
1152 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
1153 end = start + int128_get64(section->size) - 1;
1154 subpage_register(subpage, start, end,
1155 phys_section_add(&d->map, section));
1159 static void register_multipage(AddressSpaceDispatch *d,
1160 MemoryRegionSection *section)
1162 hwaddr start_addr = section->offset_within_address_space;
1163 uint16_t section_index = phys_section_add(&d->map, section);
1164 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1168 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
1171 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
1173 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1174 AddressSpaceDispatch *d = as->next_dispatch;
1175 MemoryRegionSection now = *section, remain = *section;
1176 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
1178 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
1179 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
1180 - now.offset_within_address_space;
1182 now.size = int128_min(int128_make64(left), now.size);
1183 register_subpage(d, &now);
1185 now.size = int128_zero();
1187 while (int128_ne(remain.size, now.size)) {
1188 remain.size = int128_sub(remain.size, now.size);
1189 remain.offset_within_address_space += int128_get64(now.size);
1190 remain.offset_within_region += int128_get64(now.size);
1192 if (int128_lt(remain.size, page_size)) {
1193 register_subpage(d, &now);
1194 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1195 now.size = page_size;
1196 register_subpage(d, &now);
1198 now.size = int128_and(now.size, int128_neg(page_size));
1199 register_multipage(d, &now);
1204 void qemu_flush_coalesced_mmio_buffer(void)
1207 kvm_flush_coalesced_mmio_buffer();
1210 void qemu_mutex_lock_ramlist(void)
1212 qemu_mutex_lock(&ram_list.mutex);
1215 void qemu_mutex_unlock_ramlist(void)
1217 qemu_mutex_unlock(&ram_list.mutex);
1221 static void *file_ram_alloc(RAMBlock *block,
1226 bool unlink_on_error = false;
1228 char *sanitized_name;
1230 void *area = MAP_FAILED;
1234 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1236 "host lacks kvm mmu notifiers, -mem-path unsupported");
1241 fd = open(path, O_RDWR);
1243 /* @path names an existing file, use it */
1246 if (errno == ENOENT) {
1247 /* @path names a file that doesn't exist, create it */
1248 fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
1250 unlink_on_error = true;
1253 } else if (errno == EISDIR) {
1254 /* @path names a directory, create a file there */
1255 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1256 sanitized_name = g_strdup(memory_region_name(block->mr));
1257 for (c = sanitized_name; *c != '\0'; c++) {
1263 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1265 g_free(sanitized_name);
1267 fd = mkstemp(filename);
1275 if (errno != EEXIST && errno != EINTR) {
1276 error_setg_errno(errp, errno,
1277 "can't open backing store %s for guest RAM",
1282 * Try again on EINTR and EEXIST. The latter happens when
1283 * something else creates the file between our two open().
1287 page_size = qemu_fd_getpagesize(fd);
1288 block->mr->align = MAX(page_size, QEMU_VMALLOC_ALIGN);
1290 if (memory < page_size) {
1291 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1292 "or larger than page size 0x%" PRIx64,
1297 memory = ROUND_UP(memory, page_size);
1300 * ftruncate is not supported by hugetlbfs in older
1301 * hosts, so don't bother bailing out on errors.
1302 * If anything goes wrong with it under other filesystems,
1305 if (ftruncate(fd, memory)) {
1306 perror("ftruncate");
1309 area = qemu_ram_mmap(fd, memory, block->mr->align,
1310 block->flags & RAM_SHARED);
1311 if (area == MAP_FAILED) {
1312 error_setg_errno(errp, errno,
1313 "unable to map backing store for guest RAM");
1318 os_mem_prealloc(fd, area, memory, errp);
1319 if (errp && *errp) {
1328 if (area != MAP_FAILED) {
1329 qemu_ram_munmap(area, memory);
1331 if (unlink_on_error) {
1341 /* Called with the ramlist lock held. */
1342 static ram_addr_t find_ram_offset(ram_addr_t size)
1344 RAMBlock *block, *next_block;
1345 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1347 assert(size != 0); /* it would hand out same offset multiple times */
1349 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
1353 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1354 ram_addr_t end, next = RAM_ADDR_MAX;
1356 end = block->offset + block->max_length;
1358 QLIST_FOREACH_RCU(next_block, &ram_list.blocks, next) {
1359 if (next_block->offset >= end) {
1360 next = MIN(next, next_block->offset);
1363 if (next - end >= size && next - end < mingap) {
1365 mingap = next - end;
1369 if (offset == RAM_ADDR_MAX) {
1370 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1378 ram_addr_t last_ram_offset(void)
1381 ram_addr_t last = 0;
1384 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1385 last = MAX(last, block->offset + block->max_length);
1391 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1395 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1396 if (!machine_dump_guest_core(current_machine)) {
1397 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1399 perror("qemu_madvise");
1400 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1401 "but dump_guest_core=off specified\n");
1406 const char *qemu_ram_get_idstr(RAMBlock *rb)
1411 /* Called with iothread lock held. */
1412 void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
1417 assert(!new_block->idstr[0]);
1420 char *id = qdev_get_dev_path(dev);
1422 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1426 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1429 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1430 if (block != new_block &&
1431 !strcmp(block->idstr, new_block->idstr)) {
1432 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1440 /* Called with iothread lock held. */
1441 void qemu_ram_unset_idstr(RAMBlock *block)
1443 /* FIXME: arch_init.c assumes that this is not called throughout
1444 * migration. Ignore the problem since hot-unplug during migration
1445 * does not work anyway.
1448 memset(block->idstr, 0, sizeof(block->idstr));
1452 static int memory_try_enable_merging(void *addr, size_t len)
1454 if (!machine_mem_merge(current_machine)) {
1455 /* disabled by the user */
1459 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1462 /* Only legal before guest might have detected the memory size: e.g. on
1463 * incoming migration, or right after reset.
1465 * As memory core doesn't know how is memory accessed, it is up to
1466 * resize callback to update device state and/or add assertions to detect
1467 * misuse, if necessary.
1469 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
1473 newsize = HOST_PAGE_ALIGN(newsize);
1475 if (block->used_length == newsize) {
1479 if (!(block->flags & RAM_RESIZEABLE)) {
1480 error_setg_errno(errp, EINVAL,
1481 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1482 " in != 0x" RAM_ADDR_FMT, block->idstr,
1483 newsize, block->used_length);
1487 if (block->max_length < newsize) {
1488 error_setg_errno(errp, EINVAL,
1489 "Length too large: %s: 0x" RAM_ADDR_FMT
1490 " > 0x" RAM_ADDR_FMT, block->idstr,
1491 newsize, block->max_length);
1495 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1496 block->used_length = newsize;
1497 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1499 memory_region_set_size(block->mr, newsize);
1500 if (block->resized) {
1501 block->resized(block->idstr, newsize, block->host);
1506 /* Called with ram_list.mutex held */
1507 static void dirty_memory_extend(ram_addr_t old_ram_size,
1508 ram_addr_t new_ram_size)
1510 ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
1511 DIRTY_MEMORY_BLOCK_SIZE);
1512 ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
1513 DIRTY_MEMORY_BLOCK_SIZE);
1516 /* Only need to extend if block count increased */
1517 if (new_num_blocks <= old_num_blocks) {
1521 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1522 DirtyMemoryBlocks *old_blocks;
1523 DirtyMemoryBlocks *new_blocks;
1526 old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
1527 new_blocks = g_malloc(sizeof(*new_blocks) +
1528 sizeof(new_blocks->blocks[0]) * new_num_blocks);
1530 if (old_num_blocks) {
1531 memcpy(new_blocks->blocks, old_blocks->blocks,
1532 old_num_blocks * sizeof(old_blocks->blocks[0]));
1535 for (j = old_num_blocks; j < new_num_blocks; j++) {
1536 new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
1539 atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
1542 g_free_rcu(old_blocks, rcu);
1547 static void ram_block_add(RAMBlock *new_block, Error **errp)
1550 RAMBlock *last_block = NULL;
1551 ram_addr_t old_ram_size, new_ram_size;
1554 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1556 qemu_mutex_lock_ramlist();
1557 new_block->offset = find_ram_offset(new_block->max_length);
1559 if (!new_block->host) {
1560 if (xen_enabled()) {
1561 xen_ram_alloc(new_block->offset, new_block->max_length,
1562 new_block->mr, &err);
1564 error_propagate(errp, err);
1565 qemu_mutex_unlock_ramlist();
1569 new_block->host = phys_mem_alloc(new_block->max_length,
1570 &new_block->mr->align);
1571 if (!new_block->host) {
1572 error_setg_errno(errp, errno,
1573 "cannot set up guest memory '%s'",
1574 memory_region_name(new_block->mr));
1575 qemu_mutex_unlock_ramlist();
1578 memory_try_enable_merging(new_block->host, new_block->max_length);
1582 new_ram_size = MAX(old_ram_size,
1583 (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
1584 if (new_ram_size > old_ram_size) {
1585 migration_bitmap_extend(old_ram_size, new_ram_size);
1586 dirty_memory_extend(old_ram_size, new_ram_size);
1588 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1589 * QLIST (which has an RCU-friendly variant) does not have insertion at
1590 * tail, so save the last element in last_block.
1592 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1594 if (block->max_length < new_block->max_length) {
1599 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
1600 } else if (last_block) {
1601 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
1602 } else { /* list is empty */
1603 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
1605 ram_list.mru_block = NULL;
1607 /* Write list before version */
1610 qemu_mutex_unlock_ramlist();
1612 cpu_physical_memory_set_dirty_range(new_block->offset,
1613 new_block->used_length,
1616 if (new_block->host) {
1617 qemu_ram_setup_dump(new_block->host, new_block->max_length);
1618 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
1619 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
1620 if (kvm_enabled()) {
1621 kvm_setup_guest_memory(new_block->host, new_block->max_length);
1625 * In Hax, the qemu allocate the virtual address, and HAX kernel
1626 * populate the memory with physical memory. Currently we have no
1627 * paging, so user should make sure enough free memory in advance
1629 if (hax_enabled()) {
1630 int ret = hax_populate_ram((uint64_t)(uintptr_t)new_block->host,
1631 new_block->max_length);
1633 fprintf(stderr, "HAX failed to populate ram\n");
1642 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
1643 bool share, const char *mem_path,
1646 RAMBlock *new_block;
1647 Error *local_err = NULL;
1649 if (xen_enabled()) {
1650 error_setg(errp, "-mem-path not supported with Xen");
1654 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1656 * file_ram_alloc() needs to allocate just like
1657 * phys_mem_alloc, but we haven't bothered to provide
1661 "-mem-path not supported with this accelerator");
1665 size = HOST_PAGE_ALIGN(size);
1666 new_block = g_malloc0(sizeof(*new_block));
1668 new_block->used_length = size;
1669 new_block->max_length = size;
1670 new_block->flags = share ? RAM_SHARED : 0;
1671 new_block->host = file_ram_alloc(new_block, size,
1673 if (!new_block->host) {
1678 ram_block_add(new_block, &local_err);
1681 error_propagate(errp, local_err);
1689 RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
1690 void (*resized)(const char*,
1693 void *host, bool resizeable,
1694 MemoryRegion *mr, Error **errp)
1696 RAMBlock *new_block;
1697 Error *local_err = NULL;
1699 size = HOST_PAGE_ALIGN(size);
1700 max_size = HOST_PAGE_ALIGN(max_size);
1701 new_block = g_malloc0(sizeof(*new_block));
1703 new_block->resized = resized;
1704 new_block->used_length = size;
1705 new_block->max_length = max_size;
1706 assert(max_size >= size);
1708 new_block->host = host;
1710 new_block->flags |= RAM_PREALLOC;
1713 new_block->flags |= RAM_RESIZEABLE;
1715 ram_block_add(new_block, &local_err);
1718 error_propagate(errp, local_err);
1724 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1725 MemoryRegion *mr, Error **errp)
1727 return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
1730 RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
1732 return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
1735 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
1736 void (*resized)(const char*,
1739 MemoryRegion *mr, Error **errp)
1741 return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
1744 static void reclaim_ramblock(RAMBlock *block)
1746 if (block->flags & RAM_PREALLOC) {
1748 } else if (xen_enabled()) {
1749 xen_invalidate_map_cache_entry(block->host);
1751 } else if (block->fd >= 0) {
1752 qemu_ram_munmap(block->host, block->max_length);
1756 qemu_anon_ram_free(block->host, block->max_length);
1761 void qemu_ram_free(RAMBlock *block)
1767 qemu_mutex_lock_ramlist();
1768 QLIST_REMOVE_RCU(block, next);
1769 ram_list.mru_block = NULL;
1770 /* Write list before version */
1773 call_rcu(block, reclaim_ramblock, rcu);
1774 qemu_mutex_unlock_ramlist();
1778 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1785 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1786 offset = addr - block->offset;
1787 if (offset < block->max_length) {
1788 vaddr = ramblock_ptr(block, offset);
1789 if (block->flags & RAM_PREALLOC) {
1791 } else if (xen_enabled()) {
1795 if (block->fd >= 0) {
1796 flags |= (block->flags & RAM_SHARED ?
1797 MAP_SHARED : MAP_PRIVATE);
1798 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1799 flags, block->fd, offset);
1802 * Remap needs to match alloc. Accelerators that
1803 * set phys_mem_alloc never remap. If they did,
1804 * we'd need a remap hook here.
1806 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1808 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1809 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1812 if (area != vaddr) {
1813 fprintf(stderr, "Could not remap addr: "
1814 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1818 memory_try_enable_merging(vaddr, length);
1819 qemu_ram_setup_dump(vaddr, length);
1824 #endif /* !_WIN32 */
1826 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1827 * This should not be used for general purpose DMA. Use address_space_map
1828 * or address_space_rw instead. For local memory (e.g. video ram) that the
1829 * device owns, use memory_region_get_ram_ptr.
1831 * Called within RCU critical section.
1833 void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
1835 RAMBlock *block = ram_block;
1837 if (block == NULL) {
1838 block = qemu_get_ram_block(addr);
1839 addr -= block->offset;
1842 if (xen_enabled() && block->host == NULL) {
1843 /* We need to check if the requested address is in the RAM
1844 * because we don't want to map the entire memory in QEMU.
1845 * In that case just map until the end of the page.
1847 if (block->offset == 0) {
1848 return xen_map_cache(addr, 0, 0);
1851 block->host = xen_map_cache(block->offset, block->max_length, 1);
1853 return ramblock_ptr(block, addr);
1856 /* Return a host pointer to guest's ram. Similar to qemu_map_ram_ptr
1857 * but takes a size argument.
1859 * Called within RCU critical section.
1861 static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr,
1864 RAMBlock *block = ram_block;
1869 if (block == NULL) {
1870 block = qemu_get_ram_block(addr);
1871 addr -= block->offset;
1873 *size = MIN(*size, block->max_length - addr);
1875 if (xen_enabled() && block->host == NULL) {
1876 /* We need to check if the requested address is in the RAM
1877 * because we don't want to map the entire memory in QEMU.
1878 * In that case just map the requested area.
1880 if (block->offset == 0) {
1881 return xen_map_cache(addr, *size, 1);
1884 block->host = xen_map_cache(block->offset, block->max_length, 1);
1887 return ramblock_ptr(block, addr);
1891 * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
1894 * ptr: Host pointer to look up
1895 * round_offset: If true round the result offset down to a page boundary
1896 * *ram_addr: set to result ram_addr
1897 * *offset: set to result offset within the RAMBlock
1899 * Returns: RAMBlock (or NULL if not found)
1901 * By the time this function returns, the returned pointer is not protected
1902 * by RCU anymore. If the caller is not within an RCU critical section and
1903 * does not hold the iothread lock, it must have other means of protecting the
1904 * pointer, such as a reference to the region that includes the incoming
1907 RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
1911 uint8_t *host = ptr;
1913 if (xen_enabled()) {
1914 ram_addr_t ram_addr;
1916 ram_addr = xen_ram_addr_from_mapcache(ptr);
1917 block = qemu_get_ram_block(ram_addr);
1919 *offset = ram_addr - block->offset;
1926 block = atomic_rcu_read(&ram_list.mru_block);
1927 if (block && block->host && host - block->host < block->max_length) {
1931 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1932 /* This case append when the block is not mapped. */
1933 if (block->host == NULL) {
1936 if (host - block->host < block->max_length) {
1945 *offset = (host - block->host);
1947 *offset &= TARGET_PAGE_MASK;
1954 * Finds the named RAMBlock
1956 * name: The name of RAMBlock to find
1958 * Returns: RAMBlock (or NULL if not found)
1960 RAMBlock *qemu_ram_block_by_name(const char *name)
1964 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1965 if (!strcmp(name, block->idstr)) {
1973 /* Some of the softmmu routines need to translate from a host pointer
1974 (typically a TLB entry) back to a ram offset. */
1975 ram_addr_t qemu_ram_addr_from_host(void *ptr)
1980 block = qemu_ram_block_from_host(ptr, false, &offset);
1982 return RAM_ADDR_INVALID;
1985 return block->offset + offset;
1988 /* Called within RCU critical section. */
1989 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1990 uint64_t val, unsigned size)
1992 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
1993 tb_invalidate_phys_page_fast(ram_addr, size);
1997 stb_p(qemu_map_ram_ptr(NULL, ram_addr), val);
2000 stw_p(qemu_map_ram_ptr(NULL, ram_addr), val);
2003 stl_p(qemu_map_ram_ptr(NULL, ram_addr), val);
2008 /* Set both VGA and migration bits for simplicity and to remove
2009 * the notdirty callback faster.
2011 cpu_physical_memory_set_dirty_range(ram_addr, size,
2012 DIRTY_CLIENTS_NOCODE);
2013 /* we remove the notdirty callback only if the code has been
2015 if (!cpu_physical_memory_is_clean(ram_addr)) {
2016 tlb_set_dirty(current_cpu, current_cpu->mem_io_vaddr);
2020 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
2021 unsigned size, bool is_write)
2026 static const MemoryRegionOps notdirty_mem_ops = {
2027 .write = notdirty_mem_write,
2028 .valid.accepts = notdirty_mem_accepts,
2029 .endianness = DEVICE_NATIVE_ENDIAN,
2032 /* Generate a debug exception if a watchpoint has been hit. */
2033 static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
2035 CPUState *cpu = current_cpu;
2036 CPUClass *cc = CPU_GET_CLASS(cpu);
2037 CPUArchState *env = cpu->env_ptr;
2038 target_ulong pc, cs_base;
2043 if (cpu->watchpoint_hit) {
2044 /* We re-entered the check after replacing the TB. Now raise
2045 * the debug interrupt so that is will trigger after the
2046 * current instruction. */
2047 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
2050 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2051 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
2052 if (cpu_watchpoint_address_matches(wp, vaddr, len)
2053 && (wp->flags & flags)) {
2054 if (flags == BP_MEM_READ) {
2055 wp->flags |= BP_WATCHPOINT_HIT_READ;
2057 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
2059 wp->hitaddr = vaddr;
2060 wp->hitattrs = attrs;
2061 if (!cpu->watchpoint_hit) {
2062 if (wp->flags & BP_CPU &&
2063 !cc->debug_check_watchpoint(cpu, wp)) {
2064 wp->flags &= ~BP_WATCHPOINT_HIT;
2067 cpu->watchpoint_hit = wp;
2068 tb_check_watchpoint(cpu);
2069 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2070 cpu->exception_index = EXCP_DEBUG;
2073 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2074 tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
2075 cpu_loop_exit_noexc(cpu);
2079 wp->flags &= ~BP_WATCHPOINT_HIT;
2084 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2085 so these check for a hit then pass through to the normal out-of-line
2087 static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
2088 unsigned size, MemTxAttrs attrs)
2092 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2093 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
2095 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
2098 data = address_space_ldub(as, addr, attrs, &res);
2101 data = address_space_lduw(as, addr, attrs, &res);
2104 data = address_space_ldl(as, addr, attrs, &res);
2112 static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
2113 uint64_t val, unsigned size,
2117 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2118 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
2120 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
2123 address_space_stb(as, addr, val, attrs, &res);
2126 address_space_stw(as, addr, val, attrs, &res);
2129 address_space_stl(as, addr, val, attrs, &res);
2136 static const MemoryRegionOps watch_mem_ops = {
2137 .read_with_attrs = watch_mem_read,
2138 .write_with_attrs = watch_mem_write,
2139 .endianness = DEVICE_NATIVE_ENDIAN,
2142 static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2143 unsigned len, MemTxAttrs attrs)
2145 subpage_t *subpage = opaque;
2149 #if defined(DEBUG_SUBPAGE)
2150 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
2151 subpage, len, addr);
2153 res = address_space_read(subpage->as, addr + subpage->base,
2160 *data = ldub_p(buf);
2163 *data = lduw_p(buf);
2176 static MemTxResult subpage_write(void *opaque, hwaddr addr,
2177 uint64_t value, unsigned len, MemTxAttrs attrs)
2179 subpage_t *subpage = opaque;
2182 #if defined(DEBUG_SUBPAGE)
2183 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
2184 " value %"PRIx64"\n",
2185 __func__, subpage, len, addr, value);
2203 return address_space_write(subpage->as, addr + subpage->base,
2207 static bool subpage_accepts(void *opaque, hwaddr addr,
2208 unsigned len, bool is_write)
2210 subpage_t *subpage = opaque;
2211 #if defined(DEBUG_SUBPAGE)
2212 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
2213 __func__, subpage, is_write ? 'w' : 'r', len, addr);
2216 return address_space_access_valid(subpage->as, addr + subpage->base,
2220 static const MemoryRegionOps subpage_ops = {
2221 .read_with_attrs = subpage_read,
2222 .write_with_attrs = subpage_write,
2223 .impl.min_access_size = 1,
2224 .impl.max_access_size = 8,
2225 .valid.min_access_size = 1,
2226 .valid.max_access_size = 8,
2227 .valid.accepts = subpage_accepts,
2228 .endianness = DEVICE_NATIVE_ENDIAN,
2231 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2236 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2238 idx = SUBPAGE_IDX(start);
2239 eidx = SUBPAGE_IDX(end);
2240 #if defined(DEBUG_SUBPAGE)
2241 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2242 __func__, mmio, start, end, idx, eidx, section);
2244 for (; idx <= eidx; idx++) {
2245 mmio->sub_section[idx] = section;
2251 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
2255 mmio = g_malloc0(sizeof(subpage_t));
2259 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
2260 NULL, TARGET_PAGE_SIZE);
2261 mmio->iomem.subpage = true;
2262 #if defined(DEBUG_SUBPAGE)
2263 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
2264 mmio, base, TARGET_PAGE_SIZE);
2266 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
2271 static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
2275 MemoryRegionSection section = {
2276 .address_space = as,
2278 .offset_within_address_space = 0,
2279 .offset_within_region = 0,
2280 .size = int128_2_64(),
2283 return phys_section_add(map, §ion);
2286 MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index, MemTxAttrs attrs)
2288 int asidx = cpu_asidx_from_attrs(cpu, attrs);
2289 CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
2290 AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
2291 MemoryRegionSection *sections = d->map.sections;
2293 return sections[index & ~TARGET_PAGE_MASK].mr;
2296 static void io_mem_init(void)
2298 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);
2299 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
2301 memory_region_init_io(&io_mem_notdirty, NULL, ¬dirty_mem_ops, NULL,
2303 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
2307 static void mem_begin(MemoryListener *listener)
2309 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2310 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2313 n = dummy_section(&d->map, as, &io_mem_unassigned);
2314 assert(n == PHYS_SECTION_UNASSIGNED);
2315 n = dummy_section(&d->map, as, &io_mem_notdirty);
2316 assert(n == PHYS_SECTION_NOTDIRTY);
2317 n = dummy_section(&d->map, as, &io_mem_rom);
2318 assert(n == PHYS_SECTION_ROM);
2319 n = dummy_section(&d->map, as, &io_mem_watch);
2320 assert(n == PHYS_SECTION_WATCH);
2322 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
2324 as->next_dispatch = d;
2327 static void address_space_dispatch_free(AddressSpaceDispatch *d)
2329 phys_sections_free(&d->map);
2333 static void mem_commit(MemoryListener *listener)
2335 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2336 AddressSpaceDispatch *cur = as->dispatch;
2337 AddressSpaceDispatch *next = as->next_dispatch;
2339 phys_page_compact_all(next, next->map.nodes_nb);
2341 atomic_rcu_set(&as->dispatch, next);
2343 call_rcu(cur, address_space_dispatch_free, rcu);
2347 static void tcg_commit(MemoryListener *listener)
2349 CPUAddressSpace *cpuas;
2350 AddressSpaceDispatch *d;
2352 /* since each CPU stores ram addresses in its TLB cache, we must
2353 reset the modified entries */
2354 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2355 cpu_reloading_memory_map();
2356 /* The CPU and TLB are protected by the iothread lock.
2357 * We reload the dispatch pointer now because cpu_reloading_memory_map()
2358 * may have split the RCU critical section.
2360 d = atomic_rcu_read(&cpuas->as->dispatch);
2361 cpuas->memory_dispatch = d;
2362 tlb_flush(cpuas->cpu, 1);
2365 void address_space_init_dispatch(AddressSpace *as)
2367 as->dispatch = NULL;
2368 as->dispatch_listener = (MemoryListener) {
2370 .commit = mem_commit,
2371 .region_add = mem_add,
2372 .region_nop = mem_add,
2375 memory_listener_register(&as->dispatch_listener, as);
2378 void address_space_unregister(AddressSpace *as)
2380 memory_listener_unregister(&as->dispatch_listener);
2383 void address_space_destroy_dispatch(AddressSpace *as)
2385 AddressSpaceDispatch *d = as->dispatch;
2387 atomic_rcu_set(&as->dispatch, NULL);
2389 call_rcu(d, address_space_dispatch_free, rcu);
2393 static void memory_map_init(void)
2395 system_memory = g_malloc(sizeof(*system_memory));
2397 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
2398 address_space_init(&address_space_memory, system_memory, "memory");
2400 system_io = g_malloc(sizeof(*system_io));
2401 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2403 address_space_init(&address_space_io, system_io, "I/O");
2406 MemoryRegion *get_system_memory(void)
2408 return system_memory;
2411 MemoryRegion *get_system_io(void)
2416 #endif /* !defined(CONFIG_USER_ONLY) */
2418 /* physical memory access (slow version, mainly for debug) */
2419 #if defined(CONFIG_USER_ONLY)
2420 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2421 uint8_t *buf, int len, int is_write)
2428 page = addr & TARGET_PAGE_MASK;
2429 l = (page + TARGET_PAGE_SIZE) - addr;
2432 flags = page_get_flags(page);
2433 if (!(flags & PAGE_VALID))
2436 if (!(flags & PAGE_WRITE))
2438 /* XXX: this code should not depend on lock_user */
2439 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2442 unlock_user(p, addr, l);
2444 if (!(flags & PAGE_READ))
2446 /* XXX: this code should not depend on lock_user */
2447 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2450 unlock_user(p, addr, 0);
2461 static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
2464 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
2465 addr += memory_region_get_ram_addr(mr);
2467 /* No early return if dirty_log_mask is or becomes 0, because
2468 * cpu_physical_memory_set_dirty_range will still call
2469 * xen_modified_memory.
2471 if (dirty_log_mask) {
2473 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
2475 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
2476 tb_invalidate_phys_range(addr, addr + length);
2477 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2479 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
2482 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
2484 unsigned access_size_max = mr->ops->valid.max_access_size;
2486 /* Regions are assumed to support 1-4 byte accesses unless
2487 otherwise specified. */
2488 if (access_size_max == 0) {
2489 access_size_max = 4;
2492 /* Bound the maximum access by the alignment of the address. */
2493 if (!mr->ops->impl.unaligned) {
2494 unsigned align_size_max = addr & -addr;
2495 if (align_size_max != 0 && align_size_max < access_size_max) {
2496 access_size_max = align_size_max;
2500 /* Don't attempt accesses larger than the maximum. */
2501 if (l > access_size_max) {
2502 l = access_size_max;
2509 static bool prepare_mmio_access(MemoryRegion *mr)
2511 bool unlocked = !qemu_mutex_iothread_locked();
2512 bool release_lock = false;
2514 if (unlocked && mr->global_locking) {
2515 qemu_mutex_lock_iothread();
2517 release_lock = true;
2519 if (mr->flush_coalesced_mmio) {
2521 qemu_mutex_lock_iothread();
2523 qemu_flush_coalesced_mmio_buffer();
2525 qemu_mutex_unlock_iothread();
2529 return release_lock;
2532 /* Called within RCU critical section. */
2533 static MemTxResult address_space_write_continue(AddressSpace *as, hwaddr addr,
2536 int len, hwaddr addr1,
2537 hwaddr l, MemoryRegion *mr)
2541 MemTxResult result = MEMTX_OK;
2542 bool release_lock = false;
2545 if (!memory_access_is_direct(mr, true)) {
2546 release_lock |= prepare_mmio_access(mr);
2547 l = memory_access_size(mr, l, addr1);
2548 /* XXX: could force current_cpu to NULL to avoid
2552 /* 64 bit write access */
2554 result |= memory_region_dispatch_write(mr, addr1, val, 8,
2558 /* 32 bit write access */
2560 result |= memory_region_dispatch_write(mr, addr1, val, 4,
2564 /* 16 bit write access */
2566 result |= memory_region_dispatch_write(mr, addr1, val, 2,
2570 /* 8 bit write access */
2572 result |= memory_region_dispatch_write(mr, addr1, val, 1,
2580 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
2581 memcpy(ptr, buf, l);
2582 invalidate_and_set_dirty(mr, addr1, l);
2586 qemu_mutex_unlock_iothread();
2587 release_lock = false;
2599 mr = address_space_translate(as, addr, &addr1, &l, true);
2605 MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2606 const uint8_t *buf, int len)
2611 MemTxResult result = MEMTX_OK;
2616 mr = address_space_translate(as, addr, &addr1, &l, true);
2617 result = address_space_write_continue(as, addr, attrs, buf, len,
2625 /* Called within RCU critical section. */
2626 MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr,
2627 MemTxAttrs attrs, uint8_t *buf,
2628 int len, hwaddr addr1, hwaddr l,
2633 MemTxResult result = MEMTX_OK;
2634 bool release_lock = false;
2637 if (!memory_access_is_direct(mr, false)) {
2639 release_lock |= prepare_mmio_access(mr);
2640 l = memory_access_size(mr, l, addr1);
2643 /* 64 bit read access */
2644 result |= memory_region_dispatch_read(mr, addr1, &val, 8,
2649 /* 32 bit read access */
2650 result |= memory_region_dispatch_read(mr, addr1, &val, 4,
2655 /* 16 bit read access */
2656 result |= memory_region_dispatch_read(mr, addr1, &val, 2,
2661 /* 8 bit read access */
2662 result |= memory_region_dispatch_read(mr, addr1, &val, 1,
2671 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
2672 memcpy(buf, ptr, l);
2676 qemu_mutex_unlock_iothread();
2677 release_lock = false;
2689 mr = address_space_translate(as, addr, &addr1, &l, false);
2695 MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
2696 MemTxAttrs attrs, uint8_t *buf, int len)
2701 MemTxResult result = MEMTX_OK;
2706 mr = address_space_translate(as, addr, &addr1, &l, false);
2707 result = address_space_read_continue(as, addr, attrs, buf, len,
2715 MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2716 uint8_t *buf, int len, bool is_write)
2719 return address_space_write(as, addr, attrs, (uint8_t *)buf, len);
2721 return address_space_read(as, addr, attrs, (uint8_t *)buf, len);
2725 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2726 int len, int is_write)
2728 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
2729 buf, len, is_write);
2732 enum write_rom_type {
2737 static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
2738 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
2748 mr = address_space_translate(as, addr, &addr1, &l, true);
2750 if (!(memory_region_is_ram(mr) ||
2751 memory_region_is_romd(mr))) {
2752 l = memory_access_size(mr, l, addr1);
2755 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
2758 memcpy(ptr, buf, l);
2759 invalidate_and_set_dirty(mr, addr1, l);
2762 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2773 /* used for ROM loading : can write in RAM and ROM */
2774 void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
2775 const uint8_t *buf, int len)
2777 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
2780 void cpu_flush_icache_range(hwaddr start, int len)
2783 * This function should do the same thing as an icache flush that was
2784 * triggered from within the guest. For TCG we are always cache coherent,
2785 * so there is no need to flush anything. For KVM / Xen we need to flush
2786 * the host's instruction cache at least.
2788 if (tcg_enabled()) {
2792 cpu_physical_memory_write_rom_internal(&address_space_memory,
2793 start, NULL, len, FLUSH_CACHE);
2804 static BounceBuffer bounce;
2806 typedef struct MapClient {
2808 QLIST_ENTRY(MapClient) link;
2811 QemuMutex map_client_list_lock;
2812 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2813 = QLIST_HEAD_INITIALIZER(map_client_list);
2815 static void cpu_unregister_map_client_do(MapClient *client)
2817 QLIST_REMOVE(client, link);
2821 static void cpu_notify_map_clients_locked(void)
2825 while (!QLIST_EMPTY(&map_client_list)) {
2826 client = QLIST_FIRST(&map_client_list);
2827 qemu_bh_schedule(client->bh);
2828 cpu_unregister_map_client_do(client);
2832 void cpu_register_map_client(QEMUBH *bh)
2834 MapClient *client = g_malloc(sizeof(*client));
2836 qemu_mutex_lock(&map_client_list_lock);
2838 QLIST_INSERT_HEAD(&map_client_list, client, link);
2839 if (!atomic_read(&bounce.in_use)) {
2840 cpu_notify_map_clients_locked();
2842 qemu_mutex_unlock(&map_client_list_lock);
2845 void cpu_exec_init_all(void)
2847 qemu_mutex_init(&ram_list.mutex);
2850 qemu_mutex_init(&map_client_list_lock);
2853 void cpu_unregister_map_client(QEMUBH *bh)
2857 qemu_mutex_lock(&map_client_list_lock);
2858 QLIST_FOREACH(client, &map_client_list, link) {
2859 if (client->bh == bh) {
2860 cpu_unregister_map_client_do(client);
2864 qemu_mutex_unlock(&map_client_list_lock);
2867 static void cpu_notify_map_clients(void)
2869 qemu_mutex_lock(&map_client_list_lock);
2870 cpu_notify_map_clients_locked();
2871 qemu_mutex_unlock(&map_client_list_lock);
2874 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2882 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2883 if (!memory_access_is_direct(mr, is_write)) {
2884 l = memory_access_size(mr, l, addr);
2885 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2897 /* Map a physical memory region into a host virtual address.
2898 * May map a subset of the requested range, given by and returned in *plen.
2899 * May return NULL if resources needed to perform the mapping are exhausted.
2900 * Use only for reads OR writes - not for read-modify-write operations.
2901 * Use cpu_register_map_client() to know when retrying the map operation is
2902 * likely to succeed.
2904 void *address_space_map(AddressSpace *as,
2911 hwaddr l, xlat, base;
2912 MemoryRegion *mr, *this_mr;
2921 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2923 if (!memory_access_is_direct(mr, is_write)) {
2924 if (atomic_xchg(&bounce.in_use, true)) {
2928 /* Avoid unbounded allocations */
2929 l = MIN(l, TARGET_PAGE_SIZE);
2930 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2934 memory_region_ref(mr);
2937 address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED,
2943 return bounce.buffer;
2957 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2958 if (this_mr != mr || xlat != base + done) {
2963 memory_region_ref(mr);
2965 ptr = qemu_ram_ptr_length(mr->ram_block, base, plen);
2971 /* Unmaps a memory region previously mapped by address_space_map().
2972 * Will also mark the memory as dirty if is_write == 1. access_len gives
2973 * the amount of memory that was actually read or written by the caller.
2975 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2976 int is_write, hwaddr access_len)
2978 if (buffer != bounce.buffer) {
2982 mr = memory_region_from_host(buffer, &addr1);
2985 invalidate_and_set_dirty(mr, addr1, access_len);
2987 if (xen_enabled()) {
2988 xen_invalidate_map_cache_entry(buffer);
2990 memory_region_unref(mr);
2994 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
2995 bounce.buffer, access_len);
2997 qemu_vfree(bounce.buffer);
2998 bounce.buffer = NULL;
2999 memory_region_unref(bounce.mr);
3000 atomic_mb_set(&bounce.in_use, false);
3001 cpu_notify_map_clients();
3004 void *cpu_physical_memory_map(hwaddr addr,
3008 return address_space_map(&address_space_memory, addr, plen, is_write);
3011 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
3012 int is_write, hwaddr access_len)
3014 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
3017 /* warning: addr must be aligned */
3018 static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
3020 MemTxResult *result,
3021 enum device_endian endian)
3029 bool release_lock = false;
3032 mr = address_space_translate(as, addr, &addr1, &l, false);
3033 if (l < 4 || !memory_access_is_direct(mr, false)) {
3034 release_lock |= prepare_mmio_access(mr);
3037 r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
3038 #if defined(TARGET_WORDS_BIGENDIAN)
3039 if (endian == DEVICE_LITTLE_ENDIAN) {
3043 if (endian == DEVICE_BIG_ENDIAN) {
3049 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3051 case DEVICE_LITTLE_ENDIAN:
3052 val = ldl_le_p(ptr);
3054 case DEVICE_BIG_ENDIAN:
3055 val = ldl_be_p(ptr);
3067 qemu_mutex_unlock_iothread();
3073 uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
3074 MemTxAttrs attrs, MemTxResult *result)
3076 return address_space_ldl_internal(as, addr, attrs, result,
3077 DEVICE_NATIVE_ENDIAN);
3080 uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
3081 MemTxAttrs attrs, MemTxResult *result)
3083 return address_space_ldl_internal(as, addr, attrs, result,
3084 DEVICE_LITTLE_ENDIAN);
3087 uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
3088 MemTxAttrs attrs, MemTxResult *result)
3090 return address_space_ldl_internal(as, addr, attrs, result,
3094 uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
3096 return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3099 uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
3101 return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3104 uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
3106 return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3109 /* warning: addr must be aligned */
3110 static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
3112 MemTxResult *result,
3113 enum device_endian endian)
3121 bool release_lock = false;
3124 mr = address_space_translate(as, addr, &addr1, &l,
3126 if (l < 8 || !memory_access_is_direct(mr, false)) {
3127 release_lock |= prepare_mmio_access(mr);
3130 r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
3131 #if defined(TARGET_WORDS_BIGENDIAN)
3132 if (endian == DEVICE_LITTLE_ENDIAN) {
3136 if (endian == DEVICE_BIG_ENDIAN) {
3142 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3144 case DEVICE_LITTLE_ENDIAN:
3145 val = ldq_le_p(ptr);
3147 case DEVICE_BIG_ENDIAN:
3148 val = ldq_be_p(ptr);
3160 qemu_mutex_unlock_iothread();
3166 uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
3167 MemTxAttrs attrs, MemTxResult *result)
3169 return address_space_ldq_internal(as, addr, attrs, result,
3170 DEVICE_NATIVE_ENDIAN);
3173 uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
3174 MemTxAttrs attrs, MemTxResult *result)
3176 return address_space_ldq_internal(as, addr, attrs, result,
3177 DEVICE_LITTLE_ENDIAN);
3180 uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
3181 MemTxAttrs attrs, MemTxResult *result)
3183 return address_space_ldq_internal(as, addr, attrs, result,
3187 uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
3189 return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3192 uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
3194 return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3197 uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
3199 return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3203 uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
3204 MemTxAttrs attrs, MemTxResult *result)
3209 r = address_space_rw(as, addr, attrs, &val, 1, 0);
3216 uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
3218 return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3221 /* warning: addr must be aligned */
3222 static inline uint32_t address_space_lduw_internal(AddressSpace *as,
3225 MemTxResult *result,
3226 enum device_endian endian)
3234 bool release_lock = false;
3237 mr = address_space_translate(as, addr, &addr1, &l,
3239 if (l < 2 || !memory_access_is_direct(mr, false)) {
3240 release_lock |= prepare_mmio_access(mr);
3243 r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
3244 #if defined(TARGET_WORDS_BIGENDIAN)
3245 if (endian == DEVICE_LITTLE_ENDIAN) {
3249 if (endian == DEVICE_BIG_ENDIAN) {
3255 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3257 case DEVICE_LITTLE_ENDIAN:
3258 val = lduw_le_p(ptr);
3260 case DEVICE_BIG_ENDIAN:
3261 val = lduw_be_p(ptr);
3273 qemu_mutex_unlock_iothread();
3279 uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
3280 MemTxAttrs attrs, MemTxResult *result)
3282 return address_space_lduw_internal(as, addr, attrs, result,
3283 DEVICE_NATIVE_ENDIAN);
3286 uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
3287 MemTxAttrs attrs, MemTxResult *result)
3289 return address_space_lduw_internal(as, addr, attrs, result,
3290 DEVICE_LITTLE_ENDIAN);
3293 uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
3294 MemTxAttrs attrs, MemTxResult *result)
3296 return address_space_lduw_internal(as, addr, attrs, result,
3300 uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
3302 return address_space_lduw(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3305 uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
3307 return address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3310 uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
3312 return address_space_lduw_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3315 /* warning: addr must be aligned. The ram page is not masked as dirty
3316 and the code inside is not invalidated. It is useful if the dirty
3317 bits are used to track modified PTEs */
3318 void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
3319 MemTxAttrs attrs, MemTxResult *result)
3326 uint8_t dirty_log_mask;
3327 bool release_lock = false;
3330 mr = address_space_translate(as, addr, &addr1, &l,
3332 if (l < 4 || !memory_access_is_direct(mr, true)) {
3333 release_lock |= prepare_mmio_access(mr);
3335 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3337 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3340 dirty_log_mask = memory_region_get_dirty_log_mask(mr);
3341 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
3342 cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr,
3350 qemu_mutex_unlock_iothread();
3355 void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
3357 address_space_stl_notdirty(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3360 /* warning: addr must be aligned */
3361 static inline void address_space_stl_internal(AddressSpace *as,
3362 hwaddr addr, uint32_t val,
3364 MemTxResult *result,
3365 enum device_endian endian)
3372 bool release_lock = false;
3375 mr = address_space_translate(as, addr, &addr1, &l,
3377 if (l < 4 || !memory_access_is_direct(mr, true)) {
3378 release_lock |= prepare_mmio_access(mr);
3380 #if defined(TARGET_WORDS_BIGENDIAN)
3381 if (endian == DEVICE_LITTLE_ENDIAN) {
3385 if (endian == DEVICE_BIG_ENDIAN) {
3389 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3392 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3394 case DEVICE_LITTLE_ENDIAN:
3397 case DEVICE_BIG_ENDIAN:
3404 invalidate_and_set_dirty(mr, addr1, 4);
3411 qemu_mutex_unlock_iothread();
3416 void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val,
3417 MemTxAttrs attrs, MemTxResult *result)
3419 address_space_stl_internal(as, addr, val, attrs, result,
3420 DEVICE_NATIVE_ENDIAN);
3423 void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
3424 MemTxAttrs attrs, MemTxResult *result)
3426 address_space_stl_internal(as, addr, val, attrs, result,
3427 DEVICE_LITTLE_ENDIAN);
3430 void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
3431 MemTxAttrs attrs, MemTxResult *result)
3433 address_space_stl_internal(as, addr, val, attrs, result,
3437 void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3439 address_space_stl(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3442 void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3444 address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3447 void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3449 address_space_stl_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3453 void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
3454 MemTxAttrs attrs, MemTxResult *result)
3459 r = address_space_rw(as, addr, attrs, &v, 1, 1);
3465 void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3467 address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3470 /* warning: addr must be aligned */
3471 static inline void address_space_stw_internal(AddressSpace *as,
3472 hwaddr addr, uint32_t val,
3474 MemTxResult *result,
3475 enum device_endian endian)
3482 bool release_lock = false;
3485 mr = address_space_translate(as, addr, &addr1, &l, true);
3486 if (l < 2 || !memory_access_is_direct(mr, true)) {
3487 release_lock |= prepare_mmio_access(mr);
3489 #if defined(TARGET_WORDS_BIGENDIAN)
3490 if (endian == DEVICE_LITTLE_ENDIAN) {
3494 if (endian == DEVICE_BIG_ENDIAN) {
3498 r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
3501 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3503 case DEVICE_LITTLE_ENDIAN:
3506 case DEVICE_BIG_ENDIAN:
3513 invalidate_and_set_dirty(mr, addr1, 2);
3520 qemu_mutex_unlock_iothread();
3525 void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val,
3526 MemTxAttrs attrs, MemTxResult *result)
3528 address_space_stw_internal(as, addr, val, attrs, result,
3529 DEVICE_NATIVE_ENDIAN);
3532 void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
3533 MemTxAttrs attrs, MemTxResult *result)
3535 address_space_stw_internal(as, addr, val, attrs, result,
3536 DEVICE_LITTLE_ENDIAN);
3539 void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
3540 MemTxAttrs attrs, MemTxResult *result)
3542 address_space_stw_internal(as, addr, val, attrs, result,
3546 void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3548 address_space_stw(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3551 void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3553 address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3556 void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3558 address_space_stw_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3562 void address_space_stq(AddressSpace *as, hwaddr addr, uint64_t val,
3563 MemTxAttrs attrs, MemTxResult *result)
3567 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3573 void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val,
3574 MemTxAttrs attrs, MemTxResult *result)
3577 val = cpu_to_le64(val);
3578 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3583 void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val,
3584 MemTxAttrs attrs, MemTxResult *result)
3587 val = cpu_to_be64(val);
3588 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3594 void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3596 address_space_stq(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3599 void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3601 address_space_stq_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3604 void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3606 address_space_stq_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3609 /* virtual memory access for debug (includes writing to ROM) */
3610 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
3611 uint8_t *buf, int len, int is_write)
3621 page = addr & TARGET_PAGE_MASK;
3622 phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
3623 asidx = cpu_asidx_from_attrs(cpu, attrs);
3624 /* if no physical page mapped, return an error */
3625 if (phys_addr == -1)
3627 l = (page + TARGET_PAGE_SIZE) - addr;
3630 phys_addr += (addr & ~TARGET_PAGE_MASK);
3632 cpu_physical_memory_write_rom(cpu->cpu_ases[asidx].as,
3635 address_space_rw(cpu->cpu_ases[asidx].as, phys_addr,
3636 MEMTXATTRS_UNSPECIFIED,
3647 * Allows code that needs to deal with migration bitmaps etc to still be built
3648 * target independent.
3650 size_t qemu_target_page_bits(void)
3652 return TARGET_PAGE_BITS;
3658 * A helper function for the _utterly broken_ virtio device model to find out if
3659 * it's running on a big endian machine. Don't do this at home kids!
3661 bool target_words_bigendian(void);
3662 bool target_words_bigendian(void)
3664 #if defined(TARGET_WORDS_BIGENDIAN)
3671 #ifndef CONFIG_USER_ONLY
3672 bool cpu_physical_memory_is_io(hwaddr phys_addr)
3679 mr = address_space_translate(&address_space_memory,
3680 phys_addr, &phys_addr, &l, false);
3682 res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3687 int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
3693 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
3694 ret = func(block->idstr, block->host, block->offset,
3695 block->used_length, opaque);