4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
21 #include <sys/types.h>
25 #include "qemu-common.h"
29 #if !defined(CONFIG_USER_ONLY)
30 #include "hw/boards.h"
33 #include "qemu/osdep.h"
34 #include "sysemu/kvm.h"
35 #include "sysemu/sysemu.h"
36 #include "hw/xen/xen.h"
37 #include "qemu/timer.h"
38 #include "qemu/config-file.h"
39 #include "qemu/error-report.h"
40 #include "exec/memory.h"
41 #include "sysemu/dma.h"
42 #include "exec/address-spaces.h"
43 #if defined(CONFIG_USER_ONLY)
45 #else /* !CONFIG_USER_ONLY */
46 #include "sysemu/xen-mapcache.h"
49 #include "exec/cpu-all.h"
50 #include "qemu/rcu_queue.h"
51 #include "qemu/main-loop.h"
52 #include "translate-all.h"
53 #include "sysemu/replay.h"
55 #include "exec/memory-internal.h"
56 #include "exec/ram_addr.h"
58 #include "qemu/range.h"
60 #include "qemu/mmap-alloc.h"
63 //#define DEBUG_SUBPAGE
65 #if !defined(CONFIG_USER_ONLY)
66 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
67 * are protected by the ramlist lock.
69 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
71 static MemoryRegion *system_memory;
72 static MemoryRegion *system_io;
74 AddressSpace address_space_io;
75 AddressSpace address_space_memory;
77 MemoryRegion io_mem_rom, io_mem_notdirty;
78 static MemoryRegion io_mem_unassigned;
80 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
81 #define RAM_PREALLOC (1 << 0)
83 /* RAM is mmap-ed with MAP_SHARED */
84 #define RAM_SHARED (1 << 1)
86 /* Only a portion of RAM (used_length) is actually used, and migrated.
87 * This used_length size can change across reboots.
89 #define RAM_RESIZEABLE (1 << 2)
91 /* RAM is backed by an mmapped file.
93 #define RAM_FILE (1 << 3)
96 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
97 /* current CPU in the current thread. It is only valid inside
99 __thread CPUState *current_cpu;
100 /* 0 = Do not count executed instructions.
101 1 = Precise instruction counting.
102 2 = Adaptive rate instruction counting. */
105 #if !defined(CONFIG_USER_ONLY)
107 typedef struct PhysPageEntry PhysPageEntry;
109 struct PhysPageEntry {
110 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
112 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
116 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
118 /* Size of the L2 (and L3, etc) page tables. */
119 #define ADDR_SPACE_BITS 64
122 #define P_L2_SIZE (1 << P_L2_BITS)
124 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
126 typedef PhysPageEntry Node[P_L2_SIZE];
128 typedef struct PhysPageMap {
131 unsigned sections_nb;
132 unsigned sections_nb_alloc;
134 unsigned nodes_nb_alloc;
136 MemoryRegionSection *sections;
139 struct AddressSpaceDispatch {
142 /* This is a multi-level map on the physical address space.
143 * The bottom level has pointers to MemoryRegionSections.
145 PhysPageEntry phys_map;
150 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
151 typedef struct subpage_t {
155 uint16_t sub_section[TARGET_PAGE_SIZE];
158 #define PHYS_SECTION_UNASSIGNED 0
159 #define PHYS_SECTION_NOTDIRTY 1
160 #define PHYS_SECTION_ROM 2
161 #define PHYS_SECTION_WATCH 3
163 static void io_mem_init(void);
164 static void memory_map_init(void);
165 static void tcg_commit(MemoryListener *listener);
167 static MemoryRegion io_mem_watch;
170 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
171 * @cpu: the CPU whose AddressSpace this is
172 * @as: the AddressSpace itself
173 * @memory_dispatch: its dispatch pointer (cached, RCU protected)
174 * @tcg_as_listener: listener for tracking changes to the AddressSpace
176 struct CPUAddressSpace {
179 struct AddressSpaceDispatch *memory_dispatch;
180 MemoryListener tcg_as_listener;
185 #if !defined(CONFIG_USER_ONLY)
187 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
189 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
190 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
191 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
192 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
196 static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
203 ret = map->nodes_nb++;
205 assert(ret != PHYS_MAP_NODE_NIL);
206 assert(ret != map->nodes_nb_alloc);
208 e.skip = leaf ? 0 : 1;
209 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
210 for (i = 0; i < P_L2_SIZE; ++i) {
211 memcpy(&p[i], &e, sizeof(e));
216 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
217 hwaddr *index, hwaddr *nb, uint16_t leaf,
221 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
223 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
224 lp->ptr = phys_map_node_alloc(map, level == 0);
226 p = map->nodes[lp->ptr];
227 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
229 while (*nb && lp < &p[P_L2_SIZE]) {
230 if ((*index & (step - 1)) == 0 && *nb >= step) {
236 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
242 static void phys_page_set(AddressSpaceDispatch *d,
243 hwaddr index, hwaddr nb,
246 /* Wildly overreserve - it doesn't matter much. */
247 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
249 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
252 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
253 * and update our entry so we can skip it and go directly to the destination.
255 static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
257 unsigned valid_ptr = P_L2_SIZE;
262 if (lp->ptr == PHYS_MAP_NODE_NIL) {
267 for (i = 0; i < P_L2_SIZE; i++) {
268 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
275 phys_page_compact(&p[i], nodes, compacted);
279 /* We can only compress if there's only one child. */
284 assert(valid_ptr < P_L2_SIZE);
286 /* Don't compress if it won't fit in the # of bits we have. */
287 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
291 lp->ptr = p[valid_ptr].ptr;
292 if (!p[valid_ptr].skip) {
293 /* If our only child is a leaf, make this a leaf. */
294 /* By design, we should have made this node a leaf to begin with so we
295 * should never reach here.
296 * But since it's so simple to handle this, let's do it just in case we
301 lp->skip += p[valid_ptr].skip;
305 static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
307 DECLARE_BITMAP(compacted, nodes_nb);
309 if (d->phys_map.skip) {
310 phys_page_compact(&d->phys_map, d->map.nodes, compacted);
314 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
315 Node *nodes, MemoryRegionSection *sections)
318 hwaddr index = addr >> TARGET_PAGE_BITS;
321 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
322 if (lp.ptr == PHYS_MAP_NODE_NIL) {
323 return §ions[PHYS_SECTION_UNASSIGNED];
326 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
329 if (sections[lp.ptr].size.hi ||
330 range_covers_byte(sections[lp.ptr].offset_within_address_space,
331 sections[lp.ptr].size.lo, addr)) {
332 return §ions[lp.ptr];
334 return §ions[PHYS_SECTION_UNASSIGNED];
338 bool memory_region_is_unassigned(MemoryRegion *mr)
340 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
341 && mr != &io_mem_watch;
344 /* Called from RCU critical section */
345 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
347 bool resolve_subpage)
349 MemoryRegionSection *section;
352 section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
353 if (resolve_subpage && section->mr->subpage) {
354 subpage = container_of(section->mr, subpage_t, iomem);
355 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
360 /* Called from RCU critical section */
361 static MemoryRegionSection *
362 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
363 hwaddr *plen, bool resolve_subpage)
365 MemoryRegionSection *section;
369 section = address_space_lookup_region(d, addr, resolve_subpage);
370 /* Compute offset within MemoryRegionSection */
371 addr -= section->offset_within_address_space;
373 /* Compute offset within MemoryRegion */
374 *xlat = addr + section->offset_within_region;
378 /* MMIO registers can be expected to perform full-width accesses based only
379 * on their address, without considering adjacent registers that could
380 * decode to completely different MemoryRegions. When such registers
381 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
382 * regions overlap wildly. For this reason we cannot clamp the accesses
385 * If the length is small (as is the case for address_space_ldl/stl),
386 * everything works fine. If the incoming length is large, however,
387 * the caller really has to do the clamping through memory_access_size.
389 if (memory_region_is_ram(mr)) {
390 diff = int128_sub(section->size, int128_make64(addr));
391 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
396 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
398 if (memory_region_is_ram(mr)) {
399 return !(is_write && mr->readonly);
401 if (memory_region_is_romd(mr)) {
408 /* Called from RCU critical section */
409 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
410 hwaddr *xlat, hwaddr *plen,
414 MemoryRegionSection *section;
418 AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
419 section = address_space_translate_internal(d, addr, &addr, plen, true);
422 if (!mr->iommu_ops) {
426 iotlb = mr->iommu_ops->translate(mr, addr, is_write);
427 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
428 | (addr & iotlb.addr_mask));
429 *plen = MIN(*plen, (addr | iotlb.addr_mask) - addr + 1);
430 if (!(iotlb.perm & (1 << is_write))) {
431 mr = &io_mem_unassigned;
435 as = iotlb.target_as;
438 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
439 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
440 *plen = MIN(page, *plen);
447 /* Called from RCU critical section */
448 MemoryRegionSection *
449 address_space_translate_for_iotlb(CPUState *cpu, hwaddr addr,
450 hwaddr *xlat, hwaddr *plen)
452 MemoryRegionSection *section;
453 section = address_space_translate_internal(cpu->cpu_ases[0].memory_dispatch,
454 addr, xlat, plen, false);
456 assert(!section->mr->iommu_ops);
461 #if !defined(CONFIG_USER_ONLY)
463 static int cpu_common_post_load(void *opaque, int version_id)
465 CPUState *cpu = opaque;
467 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
468 version_id is increased. */
469 cpu->interrupt_request &= ~0x01;
475 static int cpu_common_pre_load(void *opaque)
477 CPUState *cpu = opaque;
479 cpu->exception_index = -1;
484 static bool cpu_common_exception_index_needed(void *opaque)
486 CPUState *cpu = opaque;
488 return tcg_enabled() && cpu->exception_index != -1;
491 static const VMStateDescription vmstate_cpu_common_exception_index = {
492 .name = "cpu_common/exception_index",
494 .minimum_version_id = 1,
495 .needed = cpu_common_exception_index_needed,
496 .fields = (VMStateField[]) {
497 VMSTATE_INT32(exception_index, CPUState),
498 VMSTATE_END_OF_LIST()
502 static bool cpu_common_crash_occurred_needed(void *opaque)
504 CPUState *cpu = opaque;
506 return cpu->crash_occurred;
509 static const VMStateDescription vmstate_cpu_common_crash_occurred = {
510 .name = "cpu_common/crash_occurred",
512 .minimum_version_id = 1,
513 .needed = cpu_common_crash_occurred_needed,
514 .fields = (VMStateField[]) {
515 VMSTATE_BOOL(crash_occurred, CPUState),
516 VMSTATE_END_OF_LIST()
520 const VMStateDescription vmstate_cpu_common = {
521 .name = "cpu_common",
523 .minimum_version_id = 1,
524 .pre_load = cpu_common_pre_load,
525 .post_load = cpu_common_post_load,
526 .fields = (VMStateField[]) {
527 VMSTATE_UINT32(halted, CPUState),
528 VMSTATE_UINT32(interrupt_request, CPUState),
529 VMSTATE_END_OF_LIST()
531 .subsections = (const VMStateDescription*[]) {
532 &vmstate_cpu_common_exception_index,
533 &vmstate_cpu_common_crash_occurred,
540 CPUState *qemu_get_cpu(int index)
545 if (cpu->cpu_index == index) {
553 #if !defined(CONFIG_USER_ONLY)
554 void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as)
556 /* We only support one address space per cpu at the moment. */
557 assert(cpu->as == as);
560 /* We've already registered the listener for our only AS */
564 cpu->cpu_ases = g_new0(CPUAddressSpace, 1);
565 cpu->cpu_ases[0].cpu = cpu;
566 cpu->cpu_ases[0].as = as;
567 cpu->cpu_ases[0].tcg_as_listener.commit = tcg_commit;
568 memory_listener_register(&cpu->cpu_ases[0].tcg_as_listener, as);
572 #ifndef CONFIG_USER_ONLY
573 static DECLARE_BITMAP(cpu_index_map, MAX_CPUMASK_BITS);
575 static int cpu_get_free_index(Error **errp)
577 int cpu = find_first_zero_bit(cpu_index_map, MAX_CPUMASK_BITS);
579 if (cpu >= MAX_CPUMASK_BITS) {
580 error_setg(errp, "Trying to use more CPUs than max of %d",
585 bitmap_set(cpu_index_map, cpu, 1);
589 void cpu_exec_exit(CPUState *cpu)
591 if (cpu->cpu_index == -1) {
592 /* cpu_index was never allocated by this @cpu or was already freed. */
596 bitmap_clear(cpu_index_map, cpu->cpu_index, 1);
601 static int cpu_get_free_index(Error **errp)
606 CPU_FOREACH(some_cpu) {
612 void cpu_exec_exit(CPUState *cpu)
617 void cpu_exec_init(CPUState *cpu, Error **errp)
619 CPUClass *cc = CPU_GET_CLASS(cpu);
621 Error *local_err = NULL;
623 #ifndef CONFIG_USER_ONLY
624 cpu->as = &address_space_memory;
625 cpu->thread_id = qemu_get_thread_id();
628 #if defined(CONFIG_USER_ONLY)
631 cpu_index = cpu->cpu_index = cpu_get_free_index(&local_err);
633 error_propagate(errp, local_err);
634 #if defined(CONFIG_USER_ONLY)
639 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
640 #if defined(CONFIG_USER_ONLY)
643 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
644 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
646 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
647 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
648 cpu_save, cpu_load, cpu->env_ptr);
649 assert(cc->vmsd == NULL);
650 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
652 if (cc->vmsd != NULL) {
653 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
657 #if defined(CONFIG_USER_ONLY)
658 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
660 tb_invalidate_phys_page_range(pc, pc + 1, 0);
663 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
665 hwaddr phys = cpu_get_phys_page_debug(cpu, pc);
667 tb_invalidate_phys_addr(cpu->as,
668 phys | (pc & ~TARGET_PAGE_MASK));
673 #if defined(CONFIG_USER_ONLY)
674 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
679 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
685 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
689 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
690 int flags, CPUWatchpoint **watchpoint)
695 /* Add a watchpoint. */
696 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
697 int flags, CPUWatchpoint **watchpoint)
701 /* forbid ranges which are empty or run off the end of the address space */
702 if (len == 0 || (addr + len - 1) < addr) {
703 error_report("tried to set invalid watchpoint at %"
704 VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
707 wp = g_malloc(sizeof(*wp));
713 /* keep all GDB-injected watchpoints in front */
714 if (flags & BP_GDB) {
715 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
717 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
720 tlb_flush_page(cpu, addr);
727 /* Remove a specific watchpoint. */
728 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
733 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
734 if (addr == wp->vaddr && len == wp->len
735 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
736 cpu_watchpoint_remove_by_ref(cpu, wp);
743 /* Remove a specific watchpoint by reference. */
744 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
746 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
748 tlb_flush_page(cpu, watchpoint->vaddr);
753 /* Remove all matching watchpoints. */
754 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
756 CPUWatchpoint *wp, *next;
758 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
759 if (wp->flags & mask) {
760 cpu_watchpoint_remove_by_ref(cpu, wp);
765 /* Return true if this watchpoint address matches the specified
766 * access (ie the address range covered by the watchpoint overlaps
767 * partially or completely with the address range covered by the
770 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
774 /* We know the lengths are non-zero, but a little caution is
775 * required to avoid errors in the case where the range ends
776 * exactly at the top of the address space and so addr + len
777 * wraps round to zero.
779 vaddr wpend = wp->vaddr + wp->len - 1;
780 vaddr addrend = addr + len - 1;
782 return !(addr > wpend || wp->vaddr > addrend);
787 /* Add a breakpoint. */
788 int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
789 CPUBreakpoint **breakpoint)
793 bp = g_malloc(sizeof(*bp));
798 /* keep all GDB-injected breakpoints in front */
799 if (flags & BP_GDB) {
800 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
802 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
805 breakpoint_invalidate(cpu, pc);
813 /* Remove a specific breakpoint. */
814 int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
818 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
819 if (bp->pc == pc && bp->flags == flags) {
820 cpu_breakpoint_remove_by_ref(cpu, bp);
827 /* Remove a specific breakpoint by reference. */
828 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
830 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
832 breakpoint_invalidate(cpu, breakpoint->pc);
837 /* Remove all matching breakpoints. */
838 void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
840 CPUBreakpoint *bp, *next;
842 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
843 if (bp->flags & mask) {
844 cpu_breakpoint_remove_by_ref(cpu, bp);
849 /* enable or disable single step mode. EXCP_DEBUG is returned by the
850 CPU loop after each instruction */
851 void cpu_single_step(CPUState *cpu, int enabled)
853 if (cpu->singlestep_enabled != enabled) {
854 cpu->singlestep_enabled = enabled;
856 kvm_update_guest_debug(cpu, 0);
858 /* must flush all the translated code to avoid inconsistencies */
859 /* XXX: only flush what is necessary */
865 void cpu_abort(CPUState *cpu, const char *fmt, ...)
872 fprintf(stderr, "qemu: fatal: ");
873 vfprintf(stderr, fmt, ap);
874 fprintf(stderr, "\n");
875 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
876 if (qemu_log_enabled()) {
877 qemu_log("qemu: fatal: ");
878 qemu_log_vprintf(fmt, ap2);
880 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
887 #if defined(CONFIG_USER_ONLY)
889 struct sigaction act;
890 sigfillset(&act.sa_mask);
891 act.sa_handler = SIG_DFL;
892 sigaction(SIGABRT, &act, NULL);
898 #if !defined(CONFIG_USER_ONLY)
899 /* Called from RCU critical section */
900 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
904 block = atomic_rcu_read(&ram_list.mru_block);
905 if (block && addr - block->offset < block->max_length) {
908 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
909 if (addr - block->offset < block->max_length) {
914 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
918 /* It is safe to write mru_block outside the iothread lock. This
923 * xxx removed from list
927 * call_rcu(reclaim_ramblock, xxx);
930 * atomic_rcu_set is not needed here. The block was already published
931 * when it was placed into the list. Here we're just making an extra
932 * copy of the pointer.
934 ram_list.mru_block = block;
938 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
945 end = TARGET_PAGE_ALIGN(start + length);
946 start &= TARGET_PAGE_MASK;
949 block = qemu_get_ram_block(start);
950 assert(block == qemu_get_ram_block(end - 1));
951 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
953 tlb_reset_dirty(cpu, start1, length);
958 /* Note: start and end must be within the same ram block. */
959 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
963 unsigned long end, page;
970 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
971 page = start >> TARGET_PAGE_BITS;
972 dirty = bitmap_test_and_clear_atomic(ram_list.dirty_memory[client],
975 if (dirty && tcg_enabled()) {
976 tlb_reset_dirty_range_all(start, length);
982 /* Called from RCU critical section */
983 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
984 MemoryRegionSection *section,
986 hwaddr paddr, hwaddr xlat,
988 target_ulong *address)
993 if (memory_region_is_ram(section->mr)) {
995 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
997 if (!section->readonly) {
998 iotlb |= PHYS_SECTION_NOTDIRTY;
1000 iotlb |= PHYS_SECTION_ROM;
1003 AddressSpaceDispatch *d;
1005 d = atomic_rcu_read(§ion->address_space->dispatch);
1006 iotlb = section - d->map.sections;
1010 /* Make accesses to pages with watchpoints go via the
1011 watchpoint trap routines. */
1012 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
1013 if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
1014 /* Avoid trapping reads of pages with a write breakpoint. */
1015 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
1016 iotlb = PHYS_SECTION_WATCH + paddr;
1017 *address |= TLB_MMIO;
1025 #endif /* defined(CONFIG_USER_ONLY) */
1027 #if !defined(CONFIG_USER_ONLY)
1029 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1031 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
1033 static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
1034 qemu_anon_ram_alloc;
1037 * Set a custom physical guest memory alloator.
1038 * Accelerators with unusual needs may need this. Hopefully, we can
1039 * get rid of it eventually.
1041 void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))
1043 phys_mem_alloc = alloc;
1046 static uint16_t phys_section_add(PhysPageMap *map,
1047 MemoryRegionSection *section)
1049 /* The physical section number is ORed with a page-aligned
1050 * pointer to produce the iotlb entries. Thus it should
1051 * never overflow into the page-aligned value.
1053 assert(map->sections_nb < TARGET_PAGE_SIZE);
1055 if (map->sections_nb == map->sections_nb_alloc) {
1056 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1057 map->sections = g_renew(MemoryRegionSection, map->sections,
1058 map->sections_nb_alloc);
1060 map->sections[map->sections_nb] = *section;
1061 memory_region_ref(section->mr);
1062 return map->sections_nb++;
1065 static void phys_section_destroy(MemoryRegion *mr)
1067 memory_region_unref(mr);
1070 subpage_t *subpage = container_of(mr, subpage_t, iomem);
1071 object_unref(OBJECT(&subpage->iomem));
1076 static void phys_sections_free(PhysPageMap *map)
1078 while (map->sections_nb > 0) {
1079 MemoryRegionSection *section = &map->sections[--map->sections_nb];
1080 phys_section_destroy(section->mr);
1082 g_free(map->sections);
1086 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
1089 hwaddr base = section->offset_within_address_space
1091 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
1092 d->map.nodes, d->map.sections);
1093 MemoryRegionSection subsection = {
1094 .offset_within_address_space = base,
1095 .size = int128_make64(TARGET_PAGE_SIZE),
1099 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
1101 if (!(existing->mr->subpage)) {
1102 subpage = subpage_init(d->as, base);
1103 subsection.address_space = d->as;
1104 subsection.mr = &subpage->iomem;
1105 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
1106 phys_section_add(&d->map, &subsection));
1108 subpage = container_of(existing->mr, subpage_t, iomem);
1110 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
1111 end = start + int128_get64(section->size) - 1;
1112 subpage_register(subpage, start, end,
1113 phys_section_add(&d->map, section));
1117 static void register_multipage(AddressSpaceDispatch *d,
1118 MemoryRegionSection *section)
1120 hwaddr start_addr = section->offset_within_address_space;
1121 uint16_t section_index = phys_section_add(&d->map, section);
1122 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1126 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
1129 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
1131 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1132 AddressSpaceDispatch *d = as->next_dispatch;
1133 MemoryRegionSection now = *section, remain = *section;
1134 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
1136 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
1137 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
1138 - now.offset_within_address_space;
1140 now.size = int128_min(int128_make64(left), now.size);
1141 register_subpage(d, &now);
1143 now.size = int128_zero();
1145 while (int128_ne(remain.size, now.size)) {
1146 remain.size = int128_sub(remain.size, now.size);
1147 remain.offset_within_address_space += int128_get64(now.size);
1148 remain.offset_within_region += int128_get64(now.size);
1150 if (int128_lt(remain.size, page_size)) {
1151 register_subpage(d, &now);
1152 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1153 now.size = page_size;
1154 register_subpage(d, &now);
1156 now.size = int128_and(now.size, int128_neg(page_size));
1157 register_multipage(d, &now);
1162 void qemu_flush_coalesced_mmio_buffer(void)
1165 kvm_flush_coalesced_mmio_buffer();
1168 void qemu_mutex_lock_ramlist(void)
1170 qemu_mutex_lock(&ram_list.mutex);
1173 void qemu_mutex_unlock_ramlist(void)
1175 qemu_mutex_unlock(&ram_list.mutex);
1180 #include <sys/vfs.h>
1182 #define HUGETLBFS_MAGIC 0x958458f6
1184 static long gethugepagesize(const char *path, Error **errp)
1190 ret = statfs(path, &fs);
1191 } while (ret != 0 && errno == EINTR);
1194 error_setg_errno(errp, errno, "failed to get page size of file %s",
1199 if (fs.f_type != HUGETLBFS_MAGIC)
1200 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
1205 static void *file_ram_alloc(RAMBlock *block,
1211 char *sanitized_name;
1216 Error *local_err = NULL;
1218 hpagesize = gethugepagesize(path, &local_err);
1220 error_propagate(errp, local_err);
1223 block->mr->align = hpagesize;
1225 if (memory < hpagesize) {
1226 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1227 "or larger than huge page size 0x%" PRIx64,
1232 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1234 "host lacks kvm mmu notifiers, -mem-path unsupported");
1238 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1239 sanitized_name = g_strdup(memory_region_name(block->mr));
1240 for (c = sanitized_name; *c != '\0'; c++) {
1245 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1247 g_free(sanitized_name);
1249 fd = mkstemp(filename);
1251 error_setg_errno(errp, errno,
1252 "unable to create backing store for hugepages");
1259 memory = ROUND_UP(memory, hpagesize);
1262 * ftruncate is not supported by hugetlbfs in older
1263 * hosts, so don't bother bailing out on errors.
1264 * If anything goes wrong with it under other filesystems,
1267 if (ftruncate(fd, memory)) {
1268 perror("ftruncate");
1271 area = qemu_ram_mmap(fd, memory, hpagesize, block->flags & RAM_SHARED);
1272 if (area == MAP_FAILED) {
1273 error_setg_errno(errp, errno,
1274 "unable to map backing store for hugepages");
1280 os_mem_prealloc(fd, area, memory);
1288 error_report("%s", error_get_pretty(*errp));
1295 /* Called with the ramlist lock held. */
1296 static ram_addr_t find_ram_offset(ram_addr_t size)
1298 RAMBlock *block, *next_block;
1299 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1301 assert(size != 0); /* it would hand out same offset multiple times */
1303 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
1307 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1308 ram_addr_t end, next = RAM_ADDR_MAX;
1310 end = block->offset + block->max_length;
1312 QLIST_FOREACH_RCU(next_block, &ram_list.blocks, next) {
1313 if (next_block->offset >= end) {
1314 next = MIN(next, next_block->offset);
1317 if (next - end >= size && next - end < mingap) {
1319 mingap = next - end;
1323 if (offset == RAM_ADDR_MAX) {
1324 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1332 ram_addr_t last_ram_offset(void)
1335 ram_addr_t last = 0;
1338 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1339 last = MAX(last, block->offset + block->max_length);
1345 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1349 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1350 if (!machine_dump_guest_core(current_machine)) {
1351 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1353 perror("qemu_madvise");
1354 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1355 "but dump_guest_core=off specified\n");
1360 /* Called within an RCU critical section, or while the ramlist lock
1363 static RAMBlock *find_ram_block(ram_addr_t addr)
1367 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1368 if (block->offset == addr) {
1376 /* Called with iothread lock held. */
1377 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1379 RAMBlock *new_block, *block;
1382 new_block = find_ram_block(addr);
1384 assert(!new_block->idstr[0]);
1387 char *id = qdev_get_dev_path(dev);
1389 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1393 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1395 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1396 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1397 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1405 /* Called with iothread lock held. */
1406 void qemu_ram_unset_idstr(ram_addr_t addr)
1410 /* FIXME: arch_init.c assumes that this is not called throughout
1411 * migration. Ignore the problem since hot-unplug during migration
1412 * does not work anyway.
1416 block = find_ram_block(addr);
1418 memset(block->idstr, 0, sizeof(block->idstr));
1423 static int memory_try_enable_merging(void *addr, size_t len)
1425 if (!machine_mem_merge(current_machine)) {
1426 /* disabled by the user */
1430 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1433 /* Only legal before guest might have detected the memory size: e.g. on
1434 * incoming migration, or right after reset.
1436 * As memory core doesn't know how is memory accessed, it is up to
1437 * resize callback to update device state and/or add assertions to detect
1438 * misuse, if necessary.
1440 int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp)
1442 RAMBlock *block = find_ram_block(base);
1446 newsize = TARGET_PAGE_ALIGN(newsize);
1448 if (block->used_length == newsize) {
1452 if (!(block->flags & RAM_RESIZEABLE)) {
1453 error_setg_errno(errp, EINVAL,
1454 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1455 " in != 0x" RAM_ADDR_FMT, block->idstr,
1456 newsize, block->used_length);
1460 if (block->max_length < newsize) {
1461 error_setg_errno(errp, EINVAL,
1462 "Length too large: %s: 0x" RAM_ADDR_FMT
1463 " > 0x" RAM_ADDR_FMT, block->idstr,
1464 newsize, block->max_length);
1468 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1469 block->used_length = newsize;
1470 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1472 memory_region_set_size(block->mr, newsize);
1473 if (block->resized) {
1474 block->resized(block->idstr, newsize, block->host);
1479 static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
1482 RAMBlock *last_block = NULL;
1483 ram_addr_t old_ram_size, new_ram_size;
1485 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1487 qemu_mutex_lock_ramlist();
1488 new_block->offset = find_ram_offset(new_block->max_length);
1490 if (!new_block->host) {
1491 if (xen_enabled()) {
1492 xen_ram_alloc(new_block->offset, new_block->max_length,
1495 new_block->host = phys_mem_alloc(new_block->max_length,
1496 &new_block->mr->align);
1497 if (!new_block->host) {
1498 error_setg_errno(errp, errno,
1499 "cannot set up guest memory '%s'",
1500 memory_region_name(new_block->mr));
1501 qemu_mutex_unlock_ramlist();
1504 memory_try_enable_merging(new_block->host, new_block->max_length);
1508 new_ram_size = MAX(old_ram_size,
1509 (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
1510 if (new_ram_size > old_ram_size) {
1511 migration_bitmap_extend(old_ram_size, new_ram_size);
1513 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1514 * QLIST (which has an RCU-friendly variant) does not have insertion at
1515 * tail, so save the last element in last_block.
1517 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1519 if (block->max_length < new_block->max_length) {
1524 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
1525 } else if (last_block) {
1526 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
1527 } else { /* list is empty */
1528 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
1530 ram_list.mru_block = NULL;
1532 /* Write list before version */
1535 qemu_mutex_unlock_ramlist();
1537 new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1539 if (new_ram_size > old_ram_size) {
1542 /* ram_list.dirty_memory[] is protected by the iothread lock. */
1543 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1544 ram_list.dirty_memory[i] =
1545 bitmap_zero_extend(ram_list.dirty_memory[i],
1546 old_ram_size, new_ram_size);
1549 cpu_physical_memory_set_dirty_range(new_block->offset,
1550 new_block->used_length,
1553 if (new_block->host) {
1554 qemu_ram_setup_dump(new_block->host, new_block->max_length);
1555 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
1556 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
1557 if (kvm_enabled()) {
1558 kvm_setup_guest_memory(new_block->host, new_block->max_length);
1562 return new_block->offset;
1566 ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
1567 bool share, const char *mem_path,
1570 RAMBlock *new_block;
1572 Error *local_err = NULL;
1574 if (xen_enabled()) {
1575 error_setg(errp, "-mem-path not supported with Xen");
1579 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1581 * file_ram_alloc() needs to allocate just like
1582 * phys_mem_alloc, but we haven't bothered to provide
1586 "-mem-path not supported with this accelerator");
1590 size = TARGET_PAGE_ALIGN(size);
1591 new_block = g_malloc0(sizeof(*new_block));
1593 new_block->used_length = size;
1594 new_block->max_length = size;
1595 new_block->flags = share ? RAM_SHARED : 0;
1596 new_block->flags |= RAM_FILE;
1597 new_block->host = file_ram_alloc(new_block, size,
1599 if (!new_block->host) {
1604 addr = ram_block_add(new_block, &local_err);
1607 error_propagate(errp, local_err);
1615 ram_addr_t qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
1616 void (*resized)(const char*,
1619 void *host, bool resizeable,
1620 MemoryRegion *mr, Error **errp)
1622 RAMBlock *new_block;
1624 Error *local_err = NULL;
1626 size = TARGET_PAGE_ALIGN(size);
1627 max_size = TARGET_PAGE_ALIGN(max_size);
1628 new_block = g_malloc0(sizeof(*new_block));
1630 new_block->resized = resized;
1631 new_block->used_length = size;
1632 new_block->max_length = max_size;
1633 assert(max_size >= size);
1635 new_block->host = host;
1637 new_block->flags |= RAM_PREALLOC;
1640 new_block->flags |= RAM_RESIZEABLE;
1642 addr = ram_block_add(new_block, &local_err);
1645 error_propagate(errp, local_err);
1651 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1652 MemoryRegion *mr, Error **errp)
1654 return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
1657 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
1659 return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
1662 ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
1663 void (*resized)(const char*,
1666 MemoryRegion *mr, Error **errp)
1668 return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
1671 void qemu_ram_free_from_ptr(ram_addr_t addr)
1675 qemu_mutex_lock_ramlist();
1676 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1677 if (addr == block->offset) {
1678 QLIST_REMOVE_RCU(block, next);
1679 ram_list.mru_block = NULL;
1680 /* Write list before version */
1683 g_free_rcu(block, rcu);
1687 qemu_mutex_unlock_ramlist();
1690 static void reclaim_ramblock(RAMBlock *block)
1692 if (block->flags & RAM_PREALLOC) {
1694 } else if (xen_enabled()) {
1695 xen_invalidate_map_cache_entry(block->host);
1697 } else if (block->fd >= 0) {
1698 if (block->flags & RAM_FILE) {
1699 qemu_ram_munmap(block->host, block->max_length);
1701 munmap(block->host, block->max_length);
1706 qemu_anon_ram_free(block->host, block->max_length);
1711 void qemu_ram_free(ram_addr_t addr)
1715 qemu_mutex_lock_ramlist();
1716 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1717 if (addr == block->offset) {
1718 QLIST_REMOVE_RCU(block, next);
1719 ram_list.mru_block = NULL;
1720 /* Write list before version */
1723 call_rcu(block, reclaim_ramblock, rcu);
1727 qemu_mutex_unlock_ramlist();
1731 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1738 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1739 offset = addr - block->offset;
1740 if (offset < block->max_length) {
1741 vaddr = ramblock_ptr(block, offset);
1742 if (block->flags & RAM_PREALLOC) {
1744 } else if (xen_enabled()) {
1748 if (block->fd >= 0) {
1749 flags |= (block->flags & RAM_SHARED ?
1750 MAP_SHARED : MAP_PRIVATE);
1751 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1752 flags, block->fd, offset);
1755 * Remap needs to match alloc. Accelerators that
1756 * set phys_mem_alloc never remap. If they did,
1757 * we'd need a remap hook here.
1759 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1761 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1762 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1765 if (area != vaddr) {
1766 fprintf(stderr, "Could not remap addr: "
1767 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1771 memory_try_enable_merging(vaddr, length);
1772 qemu_ram_setup_dump(vaddr, length);
1777 #endif /* !_WIN32 */
1779 int qemu_get_ram_fd(ram_addr_t addr)
1785 block = qemu_get_ram_block(addr);
1791 void *qemu_get_ram_block_host_ptr(ram_addr_t addr)
1797 block = qemu_get_ram_block(addr);
1798 ptr = ramblock_ptr(block, 0);
1803 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1804 * This should not be used for general purpose DMA. Use address_space_map
1805 * or address_space_rw instead. For local memory (e.g. video ram) that the
1806 * device owns, use memory_region_get_ram_ptr.
1808 * By the time this function returns, the returned pointer is not protected
1809 * by RCU anymore. If the caller is not within an RCU critical section and
1810 * does not hold the iothread lock, it must have other means of protecting the
1811 * pointer, such as a reference to the region that includes the incoming
1814 void *qemu_get_ram_ptr(ram_addr_t addr)
1820 block = qemu_get_ram_block(addr);
1822 if (xen_enabled() && block->host == NULL) {
1823 /* We need to check if the requested address is in the RAM
1824 * because we don't want to map the entire memory in QEMU.
1825 * In that case just map until the end of the page.
1827 if (block->offset == 0) {
1828 ptr = xen_map_cache(addr, 0, 0);
1832 block->host = xen_map_cache(block->offset, block->max_length, 1);
1834 ptr = ramblock_ptr(block, addr - block->offset);
1841 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1842 * but takes a size argument.
1844 * By the time this function returns, the returned pointer is not protected
1845 * by RCU anymore. If the caller is not within an RCU critical section and
1846 * does not hold the iothread lock, it must have other means of protecting the
1847 * pointer, such as a reference to the region that includes the incoming
1850 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1856 if (xen_enabled()) {
1857 return xen_map_cache(addr, *size, 1);
1861 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1862 if (addr - block->offset < block->max_length) {
1863 if (addr - block->offset + *size > block->max_length)
1864 *size = block->max_length - addr + block->offset;
1865 ptr = ramblock_ptr(block, addr - block->offset);
1871 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1876 /* Some of the softmmu routines need to translate from a host pointer
1877 * (typically a TLB entry) back to a ram offset.
1879 * By the time this function returns, the returned pointer is not protected
1880 * by RCU anymore. If the caller is not within an RCU critical section and
1881 * does not hold the iothread lock, it must have other means of protecting the
1882 * pointer, such as a reference to the region that includes the incoming
1885 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1888 uint8_t *host = ptr;
1891 if (xen_enabled()) {
1893 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1894 mr = qemu_get_ram_block(*ram_addr)->mr;
1900 block = atomic_rcu_read(&ram_list.mru_block);
1901 if (block && block->host && host - block->host < block->max_length) {
1905 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1906 /* This case append when the block is not mapped. */
1907 if (block->host == NULL) {
1910 if (host - block->host < block->max_length) {
1919 *ram_addr = block->offset + (host - block->host);
1925 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1926 uint64_t val, unsigned size)
1928 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
1929 tb_invalidate_phys_page_fast(ram_addr, size);
1933 stb_p(qemu_get_ram_ptr(ram_addr), val);
1936 stw_p(qemu_get_ram_ptr(ram_addr), val);
1939 stl_p(qemu_get_ram_ptr(ram_addr), val);
1944 /* Set both VGA and migration bits for simplicity and to remove
1945 * the notdirty callback faster.
1947 cpu_physical_memory_set_dirty_range(ram_addr, size,
1948 DIRTY_CLIENTS_NOCODE);
1949 /* we remove the notdirty callback only if the code has been
1951 if (!cpu_physical_memory_is_clean(ram_addr)) {
1952 tlb_set_dirty(current_cpu, current_cpu->mem_io_vaddr);
1956 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1957 unsigned size, bool is_write)
1962 static const MemoryRegionOps notdirty_mem_ops = {
1963 .write = notdirty_mem_write,
1964 .valid.accepts = notdirty_mem_accepts,
1965 .endianness = DEVICE_NATIVE_ENDIAN,
1968 /* Generate a debug exception if a watchpoint has been hit. */
1969 static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
1971 CPUState *cpu = current_cpu;
1972 CPUArchState *env = cpu->env_ptr;
1973 target_ulong pc, cs_base;
1978 if (cpu->watchpoint_hit) {
1979 /* We re-entered the check after replacing the TB. Now raise
1980 * the debug interrupt so that is will trigger after the
1981 * current instruction. */
1982 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
1985 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1986 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
1987 if (cpu_watchpoint_address_matches(wp, vaddr, len)
1988 && (wp->flags & flags)) {
1989 if (flags == BP_MEM_READ) {
1990 wp->flags |= BP_WATCHPOINT_HIT_READ;
1992 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
1994 wp->hitaddr = vaddr;
1995 wp->hitattrs = attrs;
1996 if (!cpu->watchpoint_hit) {
1997 cpu->watchpoint_hit = wp;
1998 tb_check_watchpoint(cpu);
1999 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2000 cpu->exception_index = EXCP_DEBUG;
2003 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2004 tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
2005 cpu_resume_from_signal(cpu, NULL);
2009 wp->flags &= ~BP_WATCHPOINT_HIT;
2014 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2015 so these check for a hit then pass through to the normal out-of-line
2017 static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
2018 unsigned size, MemTxAttrs attrs)
2023 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
2026 data = address_space_ldub(&address_space_memory, addr, attrs, &res);
2029 data = address_space_lduw(&address_space_memory, addr, attrs, &res);
2032 data = address_space_ldl(&address_space_memory, addr, attrs, &res);
2040 static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
2041 uint64_t val, unsigned size,
2046 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
2049 address_space_stb(&address_space_memory, addr, val, attrs, &res);
2052 address_space_stw(&address_space_memory, addr, val, attrs, &res);
2055 address_space_stl(&address_space_memory, addr, val, attrs, &res);
2062 static const MemoryRegionOps watch_mem_ops = {
2063 .read_with_attrs = watch_mem_read,
2064 .write_with_attrs = watch_mem_write,
2065 .endianness = DEVICE_NATIVE_ENDIAN,
2068 static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2069 unsigned len, MemTxAttrs attrs)
2071 subpage_t *subpage = opaque;
2075 #if defined(DEBUG_SUBPAGE)
2076 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
2077 subpage, len, addr);
2079 res = address_space_read(subpage->as, addr + subpage->base,
2086 *data = ldub_p(buf);
2089 *data = lduw_p(buf);
2102 static MemTxResult subpage_write(void *opaque, hwaddr addr,
2103 uint64_t value, unsigned len, MemTxAttrs attrs)
2105 subpage_t *subpage = opaque;
2108 #if defined(DEBUG_SUBPAGE)
2109 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
2110 " value %"PRIx64"\n",
2111 __func__, subpage, len, addr, value);
2129 return address_space_write(subpage->as, addr + subpage->base,
2133 static bool subpage_accepts(void *opaque, hwaddr addr,
2134 unsigned len, bool is_write)
2136 subpage_t *subpage = opaque;
2137 #if defined(DEBUG_SUBPAGE)
2138 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
2139 __func__, subpage, is_write ? 'w' : 'r', len, addr);
2142 return address_space_access_valid(subpage->as, addr + subpage->base,
2146 static const MemoryRegionOps subpage_ops = {
2147 .read_with_attrs = subpage_read,
2148 .write_with_attrs = subpage_write,
2149 .impl.min_access_size = 1,
2150 .impl.max_access_size = 8,
2151 .valid.min_access_size = 1,
2152 .valid.max_access_size = 8,
2153 .valid.accepts = subpage_accepts,
2154 .endianness = DEVICE_NATIVE_ENDIAN,
2157 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2162 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2164 idx = SUBPAGE_IDX(start);
2165 eidx = SUBPAGE_IDX(end);
2166 #if defined(DEBUG_SUBPAGE)
2167 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2168 __func__, mmio, start, end, idx, eidx, section);
2170 for (; idx <= eidx; idx++) {
2171 mmio->sub_section[idx] = section;
2177 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
2181 mmio = g_malloc0(sizeof(subpage_t));
2185 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
2186 NULL, TARGET_PAGE_SIZE);
2187 mmio->iomem.subpage = true;
2188 #if defined(DEBUG_SUBPAGE)
2189 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
2190 mmio, base, TARGET_PAGE_SIZE);
2192 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
2197 static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
2201 MemoryRegionSection section = {
2202 .address_space = as,
2204 .offset_within_address_space = 0,
2205 .offset_within_region = 0,
2206 .size = int128_2_64(),
2209 return phys_section_add(map, §ion);
2212 MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index)
2214 CPUAddressSpace *cpuas = &cpu->cpu_ases[0];
2215 AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
2216 MemoryRegionSection *sections = d->map.sections;
2218 return sections[index & ~TARGET_PAGE_MASK].mr;
2221 static void io_mem_init(void)
2223 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);
2224 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
2226 memory_region_init_io(&io_mem_notdirty, NULL, ¬dirty_mem_ops, NULL,
2228 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
2232 static void mem_begin(MemoryListener *listener)
2234 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2235 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2238 n = dummy_section(&d->map, as, &io_mem_unassigned);
2239 assert(n == PHYS_SECTION_UNASSIGNED);
2240 n = dummy_section(&d->map, as, &io_mem_notdirty);
2241 assert(n == PHYS_SECTION_NOTDIRTY);
2242 n = dummy_section(&d->map, as, &io_mem_rom);
2243 assert(n == PHYS_SECTION_ROM);
2244 n = dummy_section(&d->map, as, &io_mem_watch);
2245 assert(n == PHYS_SECTION_WATCH);
2247 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
2249 as->next_dispatch = d;
2252 static void address_space_dispatch_free(AddressSpaceDispatch *d)
2254 phys_sections_free(&d->map);
2258 static void mem_commit(MemoryListener *listener)
2260 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2261 AddressSpaceDispatch *cur = as->dispatch;
2262 AddressSpaceDispatch *next = as->next_dispatch;
2264 phys_page_compact_all(next, next->map.nodes_nb);
2266 atomic_rcu_set(&as->dispatch, next);
2268 call_rcu(cur, address_space_dispatch_free, rcu);
2272 static void tcg_commit(MemoryListener *listener)
2274 CPUAddressSpace *cpuas;
2275 AddressSpaceDispatch *d;
2277 /* since each CPU stores ram addresses in its TLB cache, we must
2278 reset the modified entries */
2279 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2280 cpu_reloading_memory_map();
2281 /* The CPU and TLB are protected by the iothread lock.
2282 * We reload the dispatch pointer now because cpu_reloading_memory_map()
2283 * may have split the RCU critical section.
2285 d = atomic_rcu_read(&cpuas->as->dispatch);
2286 cpuas->memory_dispatch = d;
2287 tlb_flush(cpuas->cpu, 1);
2290 void address_space_init_dispatch(AddressSpace *as)
2292 as->dispatch = NULL;
2293 as->dispatch_listener = (MemoryListener) {
2295 .commit = mem_commit,
2296 .region_add = mem_add,
2297 .region_nop = mem_add,
2300 memory_listener_register(&as->dispatch_listener, as);
2303 void address_space_unregister(AddressSpace *as)
2305 memory_listener_unregister(&as->dispatch_listener);
2308 void address_space_destroy_dispatch(AddressSpace *as)
2310 AddressSpaceDispatch *d = as->dispatch;
2312 atomic_rcu_set(&as->dispatch, NULL);
2314 call_rcu(d, address_space_dispatch_free, rcu);
2318 static void memory_map_init(void)
2320 system_memory = g_malloc(sizeof(*system_memory));
2322 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
2323 address_space_init(&address_space_memory, system_memory, "memory");
2325 system_io = g_malloc(sizeof(*system_io));
2326 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2328 address_space_init(&address_space_io, system_io, "I/O");
2331 MemoryRegion *get_system_memory(void)
2333 return system_memory;
2336 MemoryRegion *get_system_io(void)
2341 #endif /* !defined(CONFIG_USER_ONLY) */
2343 /* physical memory access (slow version, mainly for debug) */
2344 #if defined(CONFIG_USER_ONLY)
2345 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2346 uint8_t *buf, int len, int is_write)
2353 page = addr & TARGET_PAGE_MASK;
2354 l = (page + TARGET_PAGE_SIZE) - addr;
2357 flags = page_get_flags(page);
2358 if (!(flags & PAGE_VALID))
2361 if (!(flags & PAGE_WRITE))
2363 /* XXX: this code should not depend on lock_user */
2364 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2367 unlock_user(p, addr, l);
2369 if (!(flags & PAGE_READ))
2371 /* XXX: this code should not depend on lock_user */
2372 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2375 unlock_user(p, addr, 0);
2386 static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
2389 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
2390 /* No early return if dirty_log_mask is or becomes 0, because
2391 * cpu_physical_memory_set_dirty_range will still call
2392 * xen_modified_memory.
2394 if (dirty_log_mask) {
2396 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
2398 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
2399 tb_invalidate_phys_range(addr, addr + length);
2400 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2402 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
2405 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
2407 unsigned access_size_max = mr->ops->valid.max_access_size;
2409 /* Regions are assumed to support 1-4 byte accesses unless
2410 otherwise specified. */
2411 if (access_size_max == 0) {
2412 access_size_max = 4;
2415 /* Bound the maximum access by the alignment of the address. */
2416 if (!mr->ops->impl.unaligned) {
2417 unsigned align_size_max = addr & -addr;
2418 if (align_size_max != 0 && align_size_max < access_size_max) {
2419 access_size_max = align_size_max;
2423 /* Don't attempt accesses larger than the maximum. */
2424 if (l > access_size_max) {
2425 l = access_size_max;
2432 static bool prepare_mmio_access(MemoryRegion *mr)
2434 bool unlocked = !qemu_mutex_iothread_locked();
2435 bool release_lock = false;
2437 if (unlocked && mr->global_locking) {
2438 qemu_mutex_lock_iothread();
2440 release_lock = true;
2442 if (mr->flush_coalesced_mmio) {
2444 qemu_mutex_lock_iothread();
2446 qemu_flush_coalesced_mmio_buffer();
2448 qemu_mutex_unlock_iothread();
2452 return release_lock;
2455 MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2456 uint8_t *buf, int len, bool is_write)
2463 MemTxResult result = MEMTX_OK;
2464 bool release_lock = false;
2469 mr = address_space_translate(as, addr, &addr1, &l, is_write);
2472 if (!memory_access_is_direct(mr, is_write)) {
2473 release_lock |= prepare_mmio_access(mr);
2474 l = memory_access_size(mr, l, addr1);
2475 /* XXX: could force current_cpu to NULL to avoid
2479 /* 64 bit write access */
2481 result |= memory_region_dispatch_write(mr, addr1, val, 8,
2485 /* 32 bit write access */
2487 result |= memory_region_dispatch_write(mr, addr1, val, 4,
2491 /* 16 bit write access */
2493 result |= memory_region_dispatch_write(mr, addr1, val, 2,
2497 /* 8 bit write access */
2499 result |= memory_region_dispatch_write(mr, addr1, val, 1,
2506 addr1 += memory_region_get_ram_addr(mr);
2508 ptr = qemu_get_ram_ptr(addr1);
2509 memcpy(ptr, buf, l);
2510 invalidate_and_set_dirty(mr, addr1, l);
2513 if (!memory_access_is_direct(mr, is_write)) {
2515 release_lock |= prepare_mmio_access(mr);
2516 l = memory_access_size(mr, l, addr1);
2519 /* 64 bit read access */
2520 result |= memory_region_dispatch_read(mr, addr1, &val, 8,
2525 /* 32 bit read access */
2526 result |= memory_region_dispatch_read(mr, addr1, &val, 4,
2531 /* 16 bit read access */
2532 result |= memory_region_dispatch_read(mr, addr1, &val, 2,
2537 /* 8 bit read access */
2538 result |= memory_region_dispatch_read(mr, addr1, &val, 1,
2547 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2548 memcpy(buf, ptr, l);
2553 qemu_mutex_unlock_iothread();
2554 release_lock = false;
2566 MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2567 const uint8_t *buf, int len)
2569 return address_space_rw(as, addr, attrs, (uint8_t *)buf, len, true);
2572 MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2573 uint8_t *buf, int len)
2575 return address_space_rw(as, addr, attrs, buf, len, false);
2579 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2580 int len, int is_write)
2582 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
2583 buf, len, is_write);
2586 enum write_rom_type {
2591 static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
2592 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
2602 mr = address_space_translate(as, addr, &addr1, &l, true);
2604 if (!(memory_region_is_ram(mr) ||
2605 memory_region_is_romd(mr))) {
2606 l = memory_access_size(mr, l, addr1);
2608 addr1 += memory_region_get_ram_addr(mr);
2610 ptr = qemu_get_ram_ptr(addr1);
2613 memcpy(ptr, buf, l);
2614 invalidate_and_set_dirty(mr, addr1, l);
2617 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2628 /* used for ROM loading : can write in RAM and ROM */
2629 void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
2630 const uint8_t *buf, int len)
2632 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
2635 void cpu_flush_icache_range(hwaddr start, int len)
2638 * This function should do the same thing as an icache flush that was
2639 * triggered from within the guest. For TCG we are always cache coherent,
2640 * so there is no need to flush anything. For KVM / Xen we need to flush
2641 * the host's instruction cache at least.
2643 if (tcg_enabled()) {
2647 cpu_physical_memory_write_rom_internal(&address_space_memory,
2648 start, NULL, len, FLUSH_CACHE);
2659 static BounceBuffer bounce;
2661 typedef struct MapClient {
2663 QLIST_ENTRY(MapClient) link;
2666 QemuMutex map_client_list_lock;
2667 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2668 = QLIST_HEAD_INITIALIZER(map_client_list);
2670 static void cpu_unregister_map_client_do(MapClient *client)
2672 QLIST_REMOVE(client, link);
2676 static void cpu_notify_map_clients_locked(void)
2680 while (!QLIST_EMPTY(&map_client_list)) {
2681 client = QLIST_FIRST(&map_client_list);
2682 qemu_bh_schedule(client->bh);
2683 cpu_unregister_map_client_do(client);
2687 void cpu_register_map_client(QEMUBH *bh)
2689 MapClient *client = g_malloc(sizeof(*client));
2691 qemu_mutex_lock(&map_client_list_lock);
2693 QLIST_INSERT_HEAD(&map_client_list, client, link);
2694 if (!atomic_read(&bounce.in_use)) {
2695 cpu_notify_map_clients_locked();
2697 qemu_mutex_unlock(&map_client_list_lock);
2700 void cpu_exec_init_all(void)
2702 qemu_mutex_init(&ram_list.mutex);
2705 qemu_mutex_init(&map_client_list_lock);
2708 void cpu_unregister_map_client(QEMUBH *bh)
2712 qemu_mutex_lock(&map_client_list_lock);
2713 QLIST_FOREACH(client, &map_client_list, link) {
2714 if (client->bh == bh) {
2715 cpu_unregister_map_client_do(client);
2719 qemu_mutex_unlock(&map_client_list_lock);
2722 static void cpu_notify_map_clients(void)
2724 qemu_mutex_lock(&map_client_list_lock);
2725 cpu_notify_map_clients_locked();
2726 qemu_mutex_unlock(&map_client_list_lock);
2729 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2737 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2738 if (!memory_access_is_direct(mr, is_write)) {
2739 l = memory_access_size(mr, l, addr);
2740 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2752 /* Map a physical memory region into a host virtual address.
2753 * May map a subset of the requested range, given by and returned in *plen.
2754 * May return NULL if resources needed to perform the mapping are exhausted.
2755 * Use only for reads OR writes - not for read-modify-write operations.
2756 * Use cpu_register_map_client() to know when retrying the map operation is
2757 * likely to succeed.
2759 void *address_space_map(AddressSpace *as,
2766 hwaddr l, xlat, base;
2767 MemoryRegion *mr, *this_mr;
2776 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2778 if (!memory_access_is_direct(mr, is_write)) {
2779 if (atomic_xchg(&bounce.in_use, true)) {
2783 /* Avoid unbounded allocations */
2784 l = MIN(l, TARGET_PAGE_SIZE);
2785 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2789 memory_region_ref(mr);
2792 address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED,
2798 return bounce.buffer;
2802 raddr = memory_region_get_ram_addr(mr);
2813 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2814 if (this_mr != mr || xlat != base + done) {
2819 memory_region_ref(mr);
2822 return qemu_ram_ptr_length(raddr + base, plen);
2825 /* Unmaps a memory region previously mapped by address_space_map().
2826 * Will also mark the memory as dirty if is_write == 1. access_len gives
2827 * the amount of memory that was actually read or written by the caller.
2829 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2830 int is_write, hwaddr access_len)
2832 if (buffer != bounce.buffer) {
2836 mr = qemu_ram_addr_from_host(buffer, &addr1);
2839 invalidate_and_set_dirty(mr, addr1, access_len);
2841 if (xen_enabled()) {
2842 xen_invalidate_map_cache_entry(buffer);
2844 memory_region_unref(mr);
2848 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
2849 bounce.buffer, access_len);
2851 qemu_vfree(bounce.buffer);
2852 bounce.buffer = NULL;
2853 memory_region_unref(bounce.mr);
2854 atomic_mb_set(&bounce.in_use, false);
2855 cpu_notify_map_clients();
2858 void *cpu_physical_memory_map(hwaddr addr,
2862 return address_space_map(&address_space_memory, addr, plen, is_write);
2865 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2866 int is_write, hwaddr access_len)
2868 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2871 /* warning: addr must be aligned */
2872 static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
2874 MemTxResult *result,
2875 enum device_endian endian)
2883 bool release_lock = false;
2886 mr = address_space_translate(as, addr, &addr1, &l, false);
2887 if (l < 4 || !memory_access_is_direct(mr, false)) {
2888 release_lock |= prepare_mmio_access(mr);
2891 r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
2892 #if defined(TARGET_WORDS_BIGENDIAN)
2893 if (endian == DEVICE_LITTLE_ENDIAN) {
2897 if (endian == DEVICE_BIG_ENDIAN) {
2903 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2907 case DEVICE_LITTLE_ENDIAN:
2908 val = ldl_le_p(ptr);
2910 case DEVICE_BIG_ENDIAN:
2911 val = ldl_be_p(ptr);
2923 qemu_mutex_unlock_iothread();
2929 uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
2930 MemTxAttrs attrs, MemTxResult *result)
2932 return address_space_ldl_internal(as, addr, attrs, result,
2933 DEVICE_NATIVE_ENDIAN);
2936 uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
2937 MemTxAttrs attrs, MemTxResult *result)
2939 return address_space_ldl_internal(as, addr, attrs, result,
2940 DEVICE_LITTLE_ENDIAN);
2943 uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
2944 MemTxAttrs attrs, MemTxResult *result)
2946 return address_space_ldl_internal(as, addr, attrs, result,
2950 uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
2952 return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
2955 uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
2957 return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
2960 uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
2962 return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
2965 /* warning: addr must be aligned */
2966 static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
2968 MemTxResult *result,
2969 enum device_endian endian)
2977 bool release_lock = false;
2980 mr = address_space_translate(as, addr, &addr1, &l,
2982 if (l < 8 || !memory_access_is_direct(mr, false)) {
2983 release_lock |= prepare_mmio_access(mr);
2986 r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
2987 #if defined(TARGET_WORDS_BIGENDIAN)
2988 if (endian == DEVICE_LITTLE_ENDIAN) {
2992 if (endian == DEVICE_BIG_ENDIAN) {
2998 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
3002 case DEVICE_LITTLE_ENDIAN:
3003 val = ldq_le_p(ptr);
3005 case DEVICE_BIG_ENDIAN:
3006 val = ldq_be_p(ptr);
3018 qemu_mutex_unlock_iothread();
3024 uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
3025 MemTxAttrs attrs, MemTxResult *result)
3027 return address_space_ldq_internal(as, addr, attrs, result,
3028 DEVICE_NATIVE_ENDIAN);
3031 uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
3032 MemTxAttrs attrs, MemTxResult *result)
3034 return address_space_ldq_internal(as, addr, attrs, result,
3035 DEVICE_LITTLE_ENDIAN);
3038 uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
3039 MemTxAttrs attrs, MemTxResult *result)
3041 return address_space_ldq_internal(as, addr, attrs, result,
3045 uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
3047 return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3050 uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
3052 return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3055 uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
3057 return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3061 uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
3062 MemTxAttrs attrs, MemTxResult *result)
3067 r = address_space_rw(as, addr, attrs, &val, 1, 0);
3074 uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
3076 return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3079 /* warning: addr must be aligned */
3080 static inline uint32_t address_space_lduw_internal(AddressSpace *as,
3083 MemTxResult *result,
3084 enum device_endian endian)
3092 bool release_lock = false;
3095 mr = address_space_translate(as, addr, &addr1, &l,
3097 if (l < 2 || !memory_access_is_direct(mr, false)) {
3098 release_lock |= prepare_mmio_access(mr);
3101 r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
3102 #if defined(TARGET_WORDS_BIGENDIAN)
3103 if (endian == DEVICE_LITTLE_ENDIAN) {
3107 if (endian == DEVICE_BIG_ENDIAN) {
3113 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
3117 case DEVICE_LITTLE_ENDIAN:
3118 val = lduw_le_p(ptr);
3120 case DEVICE_BIG_ENDIAN:
3121 val = lduw_be_p(ptr);
3133 qemu_mutex_unlock_iothread();
3139 uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
3140 MemTxAttrs attrs, MemTxResult *result)
3142 return address_space_lduw_internal(as, addr, attrs, result,
3143 DEVICE_NATIVE_ENDIAN);
3146 uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
3147 MemTxAttrs attrs, MemTxResult *result)
3149 return address_space_lduw_internal(as, addr, attrs, result,
3150 DEVICE_LITTLE_ENDIAN);
3153 uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
3154 MemTxAttrs attrs, MemTxResult *result)
3156 return address_space_lduw_internal(as, addr, attrs, result,
3160 uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
3162 return address_space_lduw(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3165 uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
3167 return address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3170 uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
3172 return address_space_lduw_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3175 /* warning: addr must be aligned. The ram page is not masked as dirty
3176 and the code inside is not invalidated. It is useful if the dirty
3177 bits are used to track modified PTEs */
3178 void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
3179 MemTxAttrs attrs, MemTxResult *result)
3186 uint8_t dirty_log_mask;
3187 bool release_lock = false;
3190 mr = address_space_translate(as, addr, &addr1, &l,
3192 if (l < 4 || !memory_access_is_direct(mr, true)) {
3193 release_lock |= prepare_mmio_access(mr);
3195 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3197 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3198 ptr = qemu_get_ram_ptr(addr1);
3201 dirty_log_mask = memory_region_get_dirty_log_mask(mr);
3202 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
3203 cpu_physical_memory_set_dirty_range(addr1, 4, dirty_log_mask);
3210 qemu_mutex_unlock_iothread();
3215 void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
3217 address_space_stl_notdirty(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3220 /* warning: addr must be aligned */
3221 static inline void address_space_stl_internal(AddressSpace *as,
3222 hwaddr addr, uint32_t val,
3224 MemTxResult *result,
3225 enum device_endian endian)
3232 bool release_lock = false;
3235 mr = address_space_translate(as, addr, &addr1, &l,
3237 if (l < 4 || !memory_access_is_direct(mr, true)) {
3238 release_lock |= prepare_mmio_access(mr);
3240 #if defined(TARGET_WORDS_BIGENDIAN)
3241 if (endian == DEVICE_LITTLE_ENDIAN) {
3245 if (endian == DEVICE_BIG_ENDIAN) {
3249 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3252 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3253 ptr = qemu_get_ram_ptr(addr1);
3255 case DEVICE_LITTLE_ENDIAN:
3258 case DEVICE_BIG_ENDIAN:
3265 invalidate_and_set_dirty(mr, addr1, 4);
3272 qemu_mutex_unlock_iothread();
3277 void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val,
3278 MemTxAttrs attrs, MemTxResult *result)
3280 address_space_stl_internal(as, addr, val, attrs, result,
3281 DEVICE_NATIVE_ENDIAN);
3284 void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
3285 MemTxAttrs attrs, MemTxResult *result)
3287 address_space_stl_internal(as, addr, val, attrs, result,
3288 DEVICE_LITTLE_ENDIAN);
3291 void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
3292 MemTxAttrs attrs, MemTxResult *result)
3294 address_space_stl_internal(as, addr, val, attrs, result,
3298 void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3300 address_space_stl(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3303 void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3305 address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3308 void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3310 address_space_stl_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3314 void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
3315 MemTxAttrs attrs, MemTxResult *result)
3320 r = address_space_rw(as, addr, attrs, &v, 1, 1);
3326 void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3328 address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3331 /* warning: addr must be aligned */
3332 static inline void address_space_stw_internal(AddressSpace *as,
3333 hwaddr addr, uint32_t val,
3335 MemTxResult *result,
3336 enum device_endian endian)
3343 bool release_lock = false;
3346 mr = address_space_translate(as, addr, &addr1, &l, true);
3347 if (l < 2 || !memory_access_is_direct(mr, true)) {
3348 release_lock |= prepare_mmio_access(mr);
3350 #if defined(TARGET_WORDS_BIGENDIAN)
3351 if (endian == DEVICE_LITTLE_ENDIAN) {
3355 if (endian == DEVICE_BIG_ENDIAN) {
3359 r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
3362 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3363 ptr = qemu_get_ram_ptr(addr1);
3365 case DEVICE_LITTLE_ENDIAN:
3368 case DEVICE_BIG_ENDIAN:
3375 invalidate_and_set_dirty(mr, addr1, 2);
3382 qemu_mutex_unlock_iothread();
3387 void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val,
3388 MemTxAttrs attrs, MemTxResult *result)
3390 address_space_stw_internal(as, addr, val, attrs, result,
3391 DEVICE_NATIVE_ENDIAN);
3394 void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
3395 MemTxAttrs attrs, MemTxResult *result)
3397 address_space_stw_internal(as, addr, val, attrs, result,
3398 DEVICE_LITTLE_ENDIAN);
3401 void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
3402 MemTxAttrs attrs, MemTxResult *result)
3404 address_space_stw_internal(as, addr, val, attrs, result,
3408 void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3410 address_space_stw(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3413 void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3415 address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3418 void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3420 address_space_stw_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3424 void address_space_stq(AddressSpace *as, hwaddr addr, uint64_t val,
3425 MemTxAttrs attrs, MemTxResult *result)
3429 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3435 void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val,
3436 MemTxAttrs attrs, MemTxResult *result)
3439 val = cpu_to_le64(val);
3440 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3445 void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val,
3446 MemTxAttrs attrs, MemTxResult *result)
3449 val = cpu_to_be64(val);
3450 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3456 void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3458 address_space_stq(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3461 void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3463 address_space_stq_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3466 void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3468 address_space_stq_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3471 /* virtual memory access for debug (includes writing to ROM) */
3472 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
3473 uint8_t *buf, int len, int is_write)
3480 page = addr & TARGET_PAGE_MASK;
3481 phys_addr = cpu_get_phys_page_debug(cpu, page);
3482 /* if no physical page mapped, return an error */
3483 if (phys_addr == -1)
3485 l = (page + TARGET_PAGE_SIZE) - addr;
3488 phys_addr += (addr & ~TARGET_PAGE_MASK);
3490 cpu_physical_memory_write_rom(cpu->as, phys_addr, buf, l);
3492 address_space_rw(cpu->as, phys_addr, MEMTXATTRS_UNSPECIFIED,
3504 * A helper function for the _utterly broken_ virtio device model to find out if
3505 * it's running on a big endian machine. Don't do this at home kids!
3507 bool target_words_bigendian(void);
3508 bool target_words_bigendian(void)
3510 #if defined(TARGET_WORDS_BIGENDIAN)
3517 #ifndef CONFIG_USER_ONLY
3518 bool cpu_physical_memory_is_io(hwaddr phys_addr)
3525 mr = address_space_translate(&address_space_memory,
3526 phys_addr, &phys_addr, &l, false);
3528 res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3533 int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
3539 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
3540 ret = func(block->idstr, block->host, block->offset,
3541 block->used_length, opaque);