1 // Copyright 2011 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef V8_HEAP_SPACES_H_
6 #define V8_HEAP_SPACES_H_
8 #include "src/allocation.h"
9 #include "src/base/atomicops.h"
10 #include "src/base/bits.h"
11 #include "src/base/platform/mutex.h"
12 #include "src/hashmap.h"
15 #include "src/utils.h"
22 // -----------------------------------------------------------------------------
25 // A JS heap consists of a young generation, an old generation, and a large
26 // object space. The young generation is divided into two semispaces. A
27 // scavenger implements Cheney's copying algorithm. The old generation is
28 // separated into a map space and an old object space. The map space contains
29 // all (and only) map objects, the rest of old objects go into the old space.
30 // The old generation is collected by a mark-sweep-compact collector.
32 // The semispaces of the young generation are contiguous. The old and map
33 // spaces consists of a list of pages. A page has a page header and an object
36 // There is a separate large object space for objects larger than
37 // Page::kMaxHeapObjectSize, so that they do not have to move during
38 // collection. The large object space is paged. Pages in large object space
39 // may be larger than the page size.
41 // A store-buffer based write barrier is used to keep track of intergenerational
42 // references. See heap/store-buffer.h.
44 // During scavenges and mark-sweep collections we sometimes (after a store
45 // buffer overflow) iterate intergenerational pointers without decoding heap
46 // object maps so if the page belongs to old pointer space or large object
47 // space it is essential to guarantee that the page does not contain any
48 // garbage pointers to new space: every pointer aligned word which satisfies
49 // the Heap::InNewSpace() predicate must be a pointer to a live heap object in
50 // new space. Thus objects in old pointer and large object spaces should have a
51 // special layout (e.g. no bare integer fields). This requirement does not
52 // apply to map space which is iterated in a special fashion. However we still
53 // require pointer fields of dead maps to be cleaned.
55 // To enable lazy cleaning of old space pages we can mark chunks of the page
56 // as being garbage. Garbage sections are marked with a special map. These
57 // sections are skipped when scanning the page, even if we are otherwise
58 // scanning without regard for object boundaries. Garbage sections are chained
59 // together to form a free list after a GC. Garbage sections created outside
60 // of GCs by object trunctation etc. may not be in the free list chain. Very
61 // small free spaces are ignored, they need only be cleaned of bogus pointers
64 // Each page may have up to one special garbage section. The start of this
65 // section is denoted by the top field in the space. The end of the section
66 // is denoted by the limit field in the space. This special garbage section
67 // is not marked with a free space map in the data. The point of this section
68 // is to enable linear allocation without having to constantly update the byte
69 // array every time the top field is updated and a new object is created. The
70 // special garbage section is not in the chain of garbage sections.
72 // Since the top and limit fields are in the space, not the page, only one page
73 // has a special garbage section, and if the top and limit are equal then there
74 // is no special garbage section.
76 // Some assertion macros used in the debugging mode.
78 #define DCHECK_PAGE_ALIGNED(address) \
79 DCHECK((OffsetFrom(address) & Page::kPageAlignmentMask) == 0)
81 #define DCHECK_OBJECT_ALIGNED(address) \
82 DCHECK((OffsetFrom(address) & kObjectAlignmentMask) == 0)
84 #define DCHECK_OBJECT_SIZE(size) \
85 DCHECK((0 < size) && (size <= Page::kMaxRegularHeapObjectSize))
87 #define DCHECK_PAGE_OFFSET(offset) \
88 DCHECK((Page::kObjectStartOffset <= offset) && (offset <= Page::kPageSize))
90 #define DCHECK_MAP_PAGE_INDEX(index) \
91 DCHECK((0 <= index) && (index <= MapSpace::kMaxMapPageIndex))
95 class MemoryAllocator;
103 typedef uint32_t CellType;
105 inline MarkBit(CellType* cell, CellType mask, bool data_only)
106 : cell_(cell), mask_(mask), data_only_(data_only) {}
108 inline CellType* cell() { return cell_; }
109 inline CellType mask() { return mask_; }
112 bool operator==(const MarkBit& other) {
113 return cell_ == other.cell_ && mask_ == other.mask_;
117 inline void Set() { *cell_ |= mask_; }
118 inline bool Get() { return (*cell_ & mask_) != 0; }
119 inline void Clear() { *cell_ &= ~mask_; }
121 inline bool data_only() { return data_only_; }
123 inline MarkBit Next() {
124 CellType new_mask = mask_ << 1;
126 return MarkBit(cell_ + 1, 1, data_only_);
128 return MarkBit(cell_, new_mask, data_only_);
135 // This boolean indicates that the object is in a data-only space with no
136 // pointers. This enables some optimizations when marking.
137 // It is expected that this field is inlined and turned into control flow
138 // at the place where the MarkBit object is created.
143 // Bitmap is a sequence of cells each containing fixed number of bits.
146 static const uint32_t kBitsPerCell = 32;
147 static const uint32_t kBitsPerCellLog2 = 5;
148 static const uint32_t kBitIndexMask = kBitsPerCell - 1;
149 static const uint32_t kBytesPerCell = kBitsPerCell / kBitsPerByte;
150 static const uint32_t kBytesPerCellLog2 = kBitsPerCellLog2 - kBitsPerByteLog2;
152 static const size_t kLength = (1 << kPageSizeBits) >> (kPointerSizeLog2);
154 static const size_t kSize =
155 (1 << kPageSizeBits) >> (kPointerSizeLog2 + kBitsPerByteLog2);
158 static int CellsForLength(int length) {
159 return (length + kBitsPerCell - 1) >> kBitsPerCellLog2;
162 int CellsCount() { return CellsForLength(kLength); }
164 static int SizeFor(int cells_count) {
165 return sizeof(MarkBit::CellType) * cells_count;
168 INLINE(static uint32_t IndexToCell(uint32_t index)) {
169 return index >> kBitsPerCellLog2;
172 INLINE(static uint32_t CellToIndex(uint32_t index)) {
173 return index << kBitsPerCellLog2;
176 INLINE(static uint32_t CellAlignIndex(uint32_t index)) {
177 return (index + kBitIndexMask) & ~kBitIndexMask;
180 INLINE(MarkBit::CellType* cells()) {
181 return reinterpret_cast<MarkBit::CellType*>(this);
184 INLINE(Address address()) { return reinterpret_cast<Address>(this); }
186 INLINE(static Bitmap* FromAddress(Address addr)) {
187 return reinterpret_cast<Bitmap*>(addr);
190 inline MarkBit MarkBitFromIndex(uint32_t index, bool data_only = false) {
191 MarkBit::CellType mask = 1 << (index & kBitIndexMask);
192 MarkBit::CellType* cell = this->cells() + (index >> kBitsPerCellLog2);
193 return MarkBit(cell, mask, data_only);
196 static inline void Clear(MemoryChunk* chunk);
198 static void PrintWord(uint32_t word, uint32_t himask = 0) {
199 for (uint32_t mask = 1; mask != 0; mask <<= 1) {
200 if ((mask & himask) != 0) PrintF("[");
201 PrintF((mask & word) ? "1" : "0");
202 if ((mask & himask) != 0) PrintF("]");
208 CellPrinter() : seq_start(0), seq_type(0), seq_length(0) {}
210 void Print(uint32_t pos, uint32_t cell) {
211 if (cell == seq_type) {
231 if (seq_length > 0) {
232 PrintF("%d: %dx%d\n", seq_start, seq_type == 0 ? 0 : 1,
233 seq_length * kBitsPerCell);
238 static bool IsSeq(uint32_t cell) { return cell == 0 || cell == 0xFFFFFFFF; }
248 for (int i = 0; i < CellsCount(); i++) {
249 printer.Print(i, cells()[i]);
256 for (int i = 0; i < CellsCount(); i++) {
257 if (cells()[i] != 0) {
269 // MemoryChunk represents a memory region owned by a specific space.
270 // It is divided into the header and the body. Chunk start is always
271 // 1MB aligned. Start of the body is aligned so it can accommodate
275 // Only works if the pointer is in the first kPageSize of the MemoryChunk.
276 static MemoryChunk* FromAddress(Address a) {
277 return reinterpret_cast<MemoryChunk*>(OffsetFrom(a) & ~kAlignmentMask);
279 static const MemoryChunk* FromAddress(const byte* a) {
280 return reinterpret_cast<const MemoryChunk*>(OffsetFrom(a) &
284 // Only works for addresses in pointer spaces, not data or code spaces.
285 static inline MemoryChunk* FromAnyPointerAddress(Heap* heap, Address addr);
287 Address address() { return reinterpret_cast<Address>(this); }
289 bool is_valid() { return address() != NULL; }
291 MemoryChunk* next_chunk() const {
292 return reinterpret_cast<MemoryChunk*>(base::Acquire_Load(&next_chunk_));
295 MemoryChunk* prev_chunk() const {
296 return reinterpret_cast<MemoryChunk*>(base::Acquire_Load(&prev_chunk_));
299 void set_next_chunk(MemoryChunk* next) {
300 base::Release_Store(&next_chunk_, reinterpret_cast<base::AtomicWord>(next));
303 void set_prev_chunk(MemoryChunk* prev) {
304 base::Release_Store(&prev_chunk_, reinterpret_cast<base::AtomicWord>(prev));
307 Space* owner() const {
308 if ((reinterpret_cast<intptr_t>(owner_) & kPageHeaderTagMask) ==
310 return reinterpret_cast<Space*>(reinterpret_cast<intptr_t>(owner_) -
317 void set_owner(Space* space) {
318 DCHECK((reinterpret_cast<intptr_t>(space) & kPageHeaderTagMask) == 0);
319 owner_ = reinterpret_cast<Address>(space) + kPageHeaderTag;
320 DCHECK((reinterpret_cast<intptr_t>(owner_) & kPageHeaderTagMask) ==
324 base::VirtualMemory* reserved_memory() { return &reservation_; }
326 void InitializeReservedMemory() { reservation_.Reset(); }
328 void set_reserved_memory(base::VirtualMemory* reservation) {
329 DCHECK_NOT_NULL(reservation);
330 reservation_.TakeControl(reservation);
333 bool scan_on_scavenge() { return IsFlagSet(SCAN_ON_SCAVENGE); }
334 void initialize_scan_on_scavenge(bool scan) {
336 SetFlag(SCAN_ON_SCAVENGE);
338 ClearFlag(SCAN_ON_SCAVENGE);
341 inline void set_scan_on_scavenge(bool scan);
343 int store_buffer_counter() { return store_buffer_counter_; }
344 void set_store_buffer_counter(int counter) {
345 store_buffer_counter_ = counter;
348 bool Contains(Address addr) {
349 return addr >= area_start() && addr < area_end();
352 // Checks whether addr can be a limit of addresses in this page.
353 // It's a limit if it's in the page, or if it's just after the
354 // last byte of the page.
355 bool ContainsLimit(Address addr) {
356 return addr >= area_start() && addr <= area_end();
359 // Every n write barrier invocations we go to runtime even though
360 // we could have handled it in generated code. This lets us check
361 // whether we have hit the limit and should do some more marking.
362 static const int kWriteBarrierCounterGranularity = 500;
364 enum MemoryChunkFlags {
367 POINTERS_TO_HERE_ARE_INTERESTING,
368 POINTERS_FROM_HERE_ARE_INTERESTING,
370 IN_FROM_SPACE, // Mutually exclusive with IN_TO_SPACE.
371 IN_TO_SPACE, // All pages in new space has one of these two set.
372 NEW_SPACE_BELOW_AGE_MARK,
374 EVACUATION_CANDIDATE,
375 RESCAN_ON_EVACUATION,
377 // WAS_SWEPT indicates that marking bits have been cleared by the sweeper,
378 // otherwise marking bits are still intact.
381 // Large objects can have a progress bar in their page header. These object
382 // are scanned in increments and will be kept black while being scanned.
383 // Even if the mutator writes to them they will be kept black and a white
384 // to grey transition is performed in the value.
387 // Last flag, keep at bottom.
388 NUM_MEMORY_CHUNK_FLAGS
392 static const int kPointersToHereAreInterestingMask =
393 1 << POINTERS_TO_HERE_ARE_INTERESTING;
395 static const int kPointersFromHereAreInterestingMask =
396 1 << POINTERS_FROM_HERE_ARE_INTERESTING;
398 static const int kEvacuationCandidateMask = 1 << EVACUATION_CANDIDATE;
400 static const int kSkipEvacuationSlotsRecordingMask =
401 (1 << EVACUATION_CANDIDATE) | (1 << RESCAN_ON_EVACUATION) |
402 (1 << IN_FROM_SPACE) | (1 << IN_TO_SPACE);
405 void SetFlag(int flag) { flags_ |= static_cast<uintptr_t>(1) << flag; }
407 void ClearFlag(int flag) { flags_ &= ~(static_cast<uintptr_t>(1) << flag); }
409 void SetFlagTo(int flag, bool value) {
417 bool IsFlagSet(int flag) {
418 return (flags_ & (static_cast<uintptr_t>(1) << flag)) != 0;
421 // Set or clear multiple flags at a time. The flags in the mask
422 // are set to the value in "flags", the rest retain the current value
424 void SetFlags(intptr_t flags, intptr_t mask) {
425 flags_ = (flags_ & ~mask) | (flags & mask);
428 // Return all current flags.
429 intptr_t GetFlags() { return flags_; }
432 // SWEEPING_DONE - The page state when sweeping is complete or sweeping must
433 // not be performed on that page.
434 // SWEEPING_FINALIZE - A sweeper thread is done sweeping this page and will
435 // not touch the page memory anymore.
436 // SWEEPING_IN_PROGRESS - This page is currently swept by a sweeper thread.
437 // SWEEPING_PENDING - This page is ready for parallel sweeping.
438 enum ParallelSweepingState {
441 SWEEPING_IN_PROGRESS,
445 ParallelSweepingState parallel_sweeping() {
446 return static_cast<ParallelSweepingState>(
447 base::Acquire_Load(¶llel_sweeping_));
450 void set_parallel_sweeping(ParallelSweepingState state) {
451 base::Release_Store(¶llel_sweeping_, state);
454 bool TryParallelSweeping() {
455 return base::Acquire_CompareAndSwap(¶llel_sweeping_, SWEEPING_PENDING,
456 SWEEPING_IN_PROGRESS) ==
460 bool SweepingCompleted() { return parallel_sweeping() <= SWEEPING_FINALIZE; }
462 // Manage live byte count (count of bytes known to be live,
463 // because they are marked black).
464 void ResetLiveBytes() {
465 if (FLAG_gc_verbose) {
466 PrintF("ResetLiveBytes:%p:%x->0\n", static_cast<void*>(this),
469 live_byte_count_ = 0;
471 void IncrementLiveBytes(int by) {
472 if (FLAG_gc_verbose) {
473 printf("UpdateLiveBytes:%p:%x%c=%x->%x\n", static_cast<void*>(this),
474 live_byte_count_, ((by < 0) ? '-' : '+'), ((by < 0) ? -by : by),
475 live_byte_count_ + by);
477 live_byte_count_ += by;
478 DCHECK_LE(static_cast<unsigned>(live_byte_count_), size_);
481 DCHECK(static_cast<unsigned>(live_byte_count_) <= size_);
482 return live_byte_count_;
485 int write_barrier_counter() {
486 return static_cast<int>(write_barrier_counter_);
489 void set_write_barrier_counter(int counter) {
490 write_barrier_counter_ = counter;
494 DCHECK(IsFlagSet(HAS_PROGRESS_BAR));
495 return progress_bar_;
498 void set_progress_bar(int progress_bar) {
499 DCHECK(IsFlagSet(HAS_PROGRESS_BAR));
500 progress_bar_ = progress_bar;
503 void ResetProgressBar() {
504 if (IsFlagSet(MemoryChunk::HAS_PROGRESS_BAR)) {
506 ClearFlag(MemoryChunk::HAS_PROGRESS_BAR);
510 bool IsLeftOfProgressBar(Object** slot) {
511 Address slot_address = reinterpret_cast<Address>(slot);
512 DCHECK(slot_address > this->address());
513 return (slot_address - (this->address() + kObjectStartOffset)) <
517 static void IncrementLiveBytesFromGC(Address address, int by) {
518 MemoryChunk::FromAddress(address)->IncrementLiveBytes(by);
521 static void IncrementLiveBytesFromMutator(Address address, int by);
523 static const intptr_t kAlignment =
524 (static_cast<uintptr_t>(1) << kPageSizeBits);
526 static const intptr_t kAlignmentMask = kAlignment - 1;
528 static const intptr_t kSizeOffset = 0;
530 static const intptr_t kLiveBytesOffset =
531 kSizeOffset + kPointerSize + kPointerSize + kPointerSize + kPointerSize +
532 kPointerSize + kPointerSize + kPointerSize + kPointerSize + kIntSize;
534 static const size_t kSlotsBufferOffset = kLiveBytesOffset + kIntSize;
536 static const size_t kWriteBarrierCounterOffset =
537 kSlotsBufferOffset + kPointerSize + kPointerSize;
539 static const size_t kHeaderSize =
540 kWriteBarrierCounterOffset + kPointerSize + kIntSize + kIntSize +
541 kPointerSize + 5 * kPointerSize + kPointerSize + kPointerSize;
543 static const int kBodyOffset =
544 CODE_POINTER_ALIGN(kHeaderSize + Bitmap::kSize);
546 // The start offset of the object area in a page. Aligned to both maps and
547 // code alignment to be suitable for both. Also aligned to 32 words because
548 // the marking bitmap is arranged in 32 bit chunks.
549 static const int kObjectStartAlignment = 32 * kPointerSize;
550 static const int kObjectStartOffset =
552 (kObjectStartAlignment - (kBodyOffset - 1) % kObjectStartAlignment);
554 size_t size() const { return size_; }
556 void set_size(size_t size) { size_ = size; }
558 void SetArea(Address area_start, Address area_end) {
559 area_start_ = area_start;
560 area_end_ = area_end;
563 Executability executable() {
564 return IsFlagSet(IS_EXECUTABLE) ? EXECUTABLE : NOT_EXECUTABLE;
567 bool ContainsOnlyData() { return IsFlagSet(CONTAINS_ONLY_DATA); }
570 return (flags_ & ((1 << IN_FROM_SPACE) | (1 << IN_TO_SPACE))) != 0;
573 bool InToSpace() { return IsFlagSet(IN_TO_SPACE); }
575 bool InFromSpace() { return IsFlagSet(IN_FROM_SPACE); }
577 // ---------------------------------------------------------------------
580 inline Bitmap* markbits() {
581 return Bitmap::FromAddress(address() + kHeaderSize);
584 void PrintMarkbits() { markbits()->Print(); }
586 inline uint32_t AddressToMarkbitIndex(Address addr) {
587 return static_cast<uint32_t>(addr - this->address()) >> kPointerSizeLog2;
590 inline static uint32_t FastAddressToMarkbitIndex(Address addr) {
591 const intptr_t offset = reinterpret_cast<intptr_t>(addr) & kAlignmentMask;
593 return static_cast<uint32_t>(offset) >> kPointerSizeLog2;
596 inline Address MarkbitIndexToAddress(uint32_t index) {
597 return this->address() + (index << kPointerSizeLog2);
600 void InsertAfter(MemoryChunk* other);
603 inline Heap* heap() const { return heap_; }
605 static const int kFlagsOffset = kPointerSize;
607 bool IsEvacuationCandidate() { return IsFlagSet(EVACUATION_CANDIDATE); }
609 bool ShouldSkipEvacuationSlotRecording() {
610 return (flags_ & kSkipEvacuationSlotsRecordingMask) != 0;
613 inline SkipList* skip_list() { return skip_list_; }
615 inline void set_skip_list(SkipList* skip_list) { skip_list_ = skip_list; }
617 inline SlotsBuffer* slots_buffer() { return slots_buffer_; }
619 inline SlotsBuffer** slots_buffer_address() { return &slots_buffer_; }
621 void MarkEvacuationCandidate() {
622 DCHECK(slots_buffer_ == NULL);
623 SetFlag(EVACUATION_CANDIDATE);
626 void ClearEvacuationCandidate() {
627 DCHECK(slots_buffer_ == NULL);
628 ClearFlag(EVACUATION_CANDIDATE);
631 Address area_start() { return area_start_; }
632 Address area_end() { return area_end_; }
633 int area_size() { return static_cast<int>(area_end() - area_start()); }
634 bool CommitArea(size_t requested);
636 // Approximate amount of physical memory committed for this chunk.
637 size_t CommittedPhysicalMemory() { return high_water_mark_; }
639 static inline void UpdateHighWaterMark(Address mark);
645 // Start and end of allocatable memory on this chunk.
649 // If the chunk needs to remember its memory reservation, it is stored here.
650 base::VirtualMemory reservation_;
651 // The identity of the owning space. This is tagged as a failure pointer, but
652 // no failure can be in an object, so this can be distinguished from any entry
656 // Used by the store buffer to keep track of which pages to mark scan-on-
658 int store_buffer_counter_;
659 // Count of bytes marked black on page.
660 int live_byte_count_;
661 SlotsBuffer* slots_buffer_;
662 SkipList* skip_list_;
663 intptr_t write_barrier_counter_;
664 // Used by the incremental marker to keep track of the scanning progress in
665 // large objects that have a progress bar and are scanned in increments.
667 // Assuming the initial allocation on a page is sequential,
668 // count highest number of bytes ever allocated on the page.
669 int high_water_mark_;
671 base::AtomicWord parallel_sweeping_;
673 // PagedSpace free-list statistics.
674 intptr_t available_in_small_free_list_;
675 intptr_t available_in_medium_free_list_;
676 intptr_t available_in_large_free_list_;
677 intptr_t available_in_huge_free_list_;
678 intptr_t non_available_small_blocks_;
680 static MemoryChunk* Initialize(Heap* heap, Address base, size_t size,
681 Address area_start, Address area_end,
682 Executability executable, Space* owner);
685 // next_chunk_ holds a pointer of type MemoryChunk
686 base::AtomicWord next_chunk_;
687 // prev_chunk_ holds a pointer of type MemoryChunk
688 base::AtomicWord prev_chunk_;
690 friend class MemoryAllocator;
694 STATIC_ASSERT(sizeof(MemoryChunk) <= MemoryChunk::kHeaderSize);
697 // -----------------------------------------------------------------------------
698 // A page is a memory chunk of a size 1MB. Large object pages may be larger.
700 // The only way to get a page pointer is by calling factory methods:
701 // Page* p = Page::FromAddress(addr); or
702 // Page* p = Page::FromAllocationTop(top);
703 class Page : public MemoryChunk {
705 // Returns the page containing a given address. The address ranges
706 // from [page_addr .. page_addr + kPageSize[
707 // This only works if the object is in fact in a page. See also MemoryChunk::
708 // FromAddress() and FromAnyAddress().
709 INLINE(static Page* FromAddress(Address a)) {
710 return reinterpret_cast<Page*>(OffsetFrom(a) & ~kPageAlignmentMask);
713 // Returns the page containing an allocation top. Because an allocation
714 // top address can be the upper bound of the page, we need to subtract
715 // it with kPointerSize first. The address ranges from
716 // [page_addr + kObjectStartOffset .. page_addr + kPageSize].
717 INLINE(static Page* FromAllocationTop(Address top)) {
718 Page* p = FromAddress(top - kPointerSize);
722 // Returns the next page in the chain of pages owned by a space.
723 inline Page* next_page();
724 inline Page* prev_page();
725 inline void set_next_page(Page* page);
726 inline void set_prev_page(Page* page);
728 // Checks whether an address is page aligned.
729 static bool IsAlignedToPageSize(Address a) {
730 return 0 == (OffsetFrom(a) & kPageAlignmentMask);
733 // Returns the offset of a given address to this page.
734 INLINE(int Offset(Address a)) {
735 int offset = static_cast<int>(a - address());
739 // Returns the address for a given offset to the this page.
740 Address OffsetToAddress(int offset) {
741 DCHECK_PAGE_OFFSET(offset);
742 return address() + offset;
745 // ---------------------------------------------------------------------
747 // Page size in bytes. This must be a multiple of the OS page size.
748 static const int kPageSize = 1 << kPageSizeBits;
750 // Maximum object size that fits in a page. Objects larger than that size
751 // are allocated in large object space and are never moved in memory. This
752 // also applies to new space allocation, since objects are never migrated
753 // from new space to large object space. Takes double alignment into account.
754 static const int kMaxRegularHeapObjectSize = kPageSize - kObjectStartOffset;
757 static const intptr_t kPageAlignmentMask = (1 << kPageSizeBits) - 1;
759 inline void ClearGCFields();
761 static inline Page* Initialize(Heap* heap, MemoryChunk* chunk,
762 Executability executable, PagedSpace* owner);
764 void InitializeAsAnchor(PagedSpace* owner);
766 bool WasSwept() { return IsFlagSet(WAS_SWEPT); }
767 void SetWasSwept() { SetFlag(WAS_SWEPT); }
768 void ClearWasSwept() { ClearFlag(WAS_SWEPT); }
770 void ResetFreeListStatistics();
772 #define FRAGMENTATION_STATS_ACCESSORS(type, name) \
773 type name() { return name##_; } \
774 void set_##name(type name) { name##_ = name; } \
775 void add_##name(type name) { name##_ += name; }
777 FRAGMENTATION_STATS_ACCESSORS(intptr_t, non_available_small_blocks)
778 FRAGMENTATION_STATS_ACCESSORS(intptr_t, available_in_small_free_list)
779 FRAGMENTATION_STATS_ACCESSORS(intptr_t, available_in_medium_free_list)
780 FRAGMENTATION_STATS_ACCESSORS(intptr_t, available_in_large_free_list)
781 FRAGMENTATION_STATS_ACCESSORS(intptr_t, available_in_huge_free_list)
783 #undef FRAGMENTATION_STATS_ACCESSORS
789 friend class MemoryAllocator;
793 STATIC_ASSERT(sizeof(Page) <= MemoryChunk::kHeaderSize);
796 class LargePage : public MemoryChunk {
798 HeapObject* GetObject() { return HeapObject::FromAddress(area_start()); }
800 inline LargePage* next_page() const {
801 return static_cast<LargePage*>(next_chunk());
804 inline void set_next_page(LargePage* page) { set_next_chunk(page); }
807 static inline LargePage* Initialize(Heap* heap, MemoryChunk* chunk);
809 friend class MemoryAllocator;
812 STATIC_ASSERT(sizeof(LargePage) <= MemoryChunk::kHeaderSize);
814 // ----------------------------------------------------------------------------
815 // Space is the abstract superclass for all allocation spaces.
816 class Space : public Malloced {
818 Space(Heap* heap, AllocationSpace id, Executability executable)
819 : heap_(heap), id_(id), executable_(executable) {}
823 Heap* heap() const { return heap_; }
825 // Does the space need executable memory?
826 Executability executable() { return executable_; }
828 // Identity used in error reporting.
829 AllocationSpace identity() { return id_; }
831 // Returns allocated size.
832 virtual intptr_t Size() = 0;
834 // Returns size of objects. Can differ from the allocated size
835 // (e.g. see LargeObjectSpace).
836 virtual intptr_t SizeOfObjects() { return Size(); }
838 virtual int RoundSizeDownToObjectAlignment(int size) {
839 if (id_ == CODE_SPACE) {
840 return RoundDown(size, kCodeAlignment);
842 return RoundDown(size, kPointerSize);
847 virtual void Print() = 0;
853 Executability executable_;
857 // ----------------------------------------------------------------------------
858 // All heap objects containing executable code (code objects) must be allocated
859 // from a 2 GB range of memory, so that they can call each other using 32-bit
860 // displacements. This happens automatically on 32-bit platforms, where 32-bit
861 // displacements cover the entire 4GB virtual address space. On 64-bit
862 // platforms, we support this using the CodeRange object, which reserves and
863 // manages a range of virtual memory.
866 explicit CodeRange(Isolate* isolate);
867 ~CodeRange() { TearDown(); }
869 // Reserves a range of virtual memory, but does not commit any of it.
870 // Can only be called once, at heap initialization time.
871 // Returns false on failure.
872 bool SetUp(size_t requested_size);
874 // Frees the range of virtual memory, and frees the data structures used to
878 bool valid() { return code_range_ != NULL; }
881 return static_cast<Address>(code_range_->address());
885 return code_range_->size();
887 bool contains(Address address) {
888 if (!valid()) return false;
889 Address start = static_cast<Address>(code_range_->address());
890 return start <= address && address < start + code_range_->size();
893 // Allocates a chunk of memory from the large-object portion of
894 // the code range. On platforms with no separate code range, should
896 MUST_USE_RESULT Address AllocateRawMemory(const size_t requested_size,
897 const size_t commit_size,
899 bool CommitRawMemory(Address start, size_t length);
900 bool UncommitRawMemory(Address start, size_t length);
901 void FreeRawMemory(Address buf, size_t length);
903 void ReserveEmergencyBlock();
904 void ReleaseEmergencyBlock();
909 // The reserved range of virtual memory that all code objects are put in.
910 base::VirtualMemory* code_range_;
911 // Plain old data class, just a struct plus a constructor.
914 FreeBlock() : start(0), size(0) {}
915 FreeBlock(Address start_arg, size_t size_arg)
916 : start(start_arg), size(size_arg) {
917 DCHECK(IsAddressAligned(start, MemoryChunk::kAlignment));
918 DCHECK(size >= static_cast<size_t>(Page::kPageSize));
920 FreeBlock(void* start_arg, size_t size_arg)
921 : start(static_cast<Address>(start_arg)), size(size_arg) {
922 DCHECK(IsAddressAligned(start, MemoryChunk::kAlignment));
923 DCHECK(size >= static_cast<size_t>(Page::kPageSize));
930 // Freed blocks of memory are added to the free list. When the allocation
931 // list is exhausted, the free list is sorted and merged to make the new
933 List<FreeBlock> free_list_;
934 // Memory is allocated from the free blocks on the allocation list.
935 // The block at current_allocation_block_index_ is the current block.
936 List<FreeBlock> allocation_list_;
937 int current_allocation_block_index_;
939 // Emergency block guarantees that we can always allocate a page for
940 // evacuation candidates when code space is compacted. Emergency block is
941 // reserved immediately after GC and is released immedietely before
942 // allocating a page for evacuation.
943 FreeBlock emergency_block_;
945 // Finds a block on the allocation list that contains at least the
946 // requested amount of memory. If none is found, sorts and merges
947 // the existing free memory blocks, and searches again.
948 // If none can be found, returns false.
949 bool GetNextAllocationBlock(size_t requested);
950 // Compares the start addresses of two free blocks.
951 static int CompareFreeBlockAddress(const FreeBlock* left,
952 const FreeBlock* right);
953 bool ReserveBlock(const size_t requested_size, FreeBlock* block);
954 void ReleaseBlock(const FreeBlock* block);
956 DISALLOW_COPY_AND_ASSIGN(CodeRange);
962 SkipList() { Clear(); }
965 for (int idx = 0; idx < kSize; idx++) {
966 starts_[idx] = reinterpret_cast<Address>(-1);
970 Address StartFor(Address addr) { return starts_[RegionNumber(addr)]; }
972 void AddObject(Address addr, int size) {
973 int start_region = RegionNumber(addr);
974 int end_region = RegionNumber(addr + size - kPointerSize);
975 for (int idx = start_region; idx <= end_region; idx++) {
976 if (starts_[idx] > addr) starts_[idx] = addr;
980 static inline int RegionNumber(Address addr) {
981 return (OffsetFrom(addr) & Page::kPageAlignmentMask) >> kRegionSizeLog2;
984 static void Update(Address addr, int size) {
985 Page* page = Page::FromAddress(addr);
986 SkipList* list = page->skip_list();
988 list = new SkipList();
989 page->set_skip_list(list);
992 list->AddObject(addr, size);
996 static const int kRegionSizeLog2 = 13;
997 static const int kRegionSize = 1 << kRegionSizeLog2;
998 static const int kSize = Page::kPageSize / kRegionSize;
1000 STATIC_ASSERT(Page::kPageSize % kRegionSize == 0);
1002 Address starts_[kSize];
1006 // ----------------------------------------------------------------------------
1007 // A space acquires chunks of memory from the operating system. The memory
1008 // allocator allocated and deallocates pages for the paged heap spaces and large
1009 // pages for large object space.
1011 // Each space has to manage it's own pages.
1013 class MemoryAllocator {
1015 explicit MemoryAllocator(Isolate* isolate);
1017 // Initializes its internal bookkeeping structures.
1018 // Max capacity of the total space and executable memory limit.
1019 bool SetUp(intptr_t max_capacity, intptr_t capacity_executable);
1023 Page* AllocatePage(intptr_t size, PagedSpace* owner,
1024 Executability executable);
1026 LargePage* AllocateLargePage(intptr_t object_size, Space* owner,
1027 Executability executable);
1029 void Free(MemoryChunk* chunk);
1031 // Returns the maximum available bytes of heaps.
1032 intptr_t Available() { return capacity_ < size_ ? 0 : capacity_ - size_; }
1034 // Returns allocated spaces in bytes.
1035 intptr_t Size() { return size_; }
1037 // Returns the maximum available executable bytes of heaps.
1038 intptr_t AvailableExecutable() {
1039 if (capacity_executable_ < size_executable_) return 0;
1040 return capacity_executable_ - size_executable_;
1043 // Returns allocated executable spaces in bytes.
1044 intptr_t SizeExecutable() { return size_executable_; }
1046 // Returns maximum available bytes that the old space can have.
1047 intptr_t MaxAvailable() {
1048 return (Available() / Page::kPageSize) * Page::kMaxRegularHeapObjectSize;
1051 // Returns an indication of whether a pointer is in a space that has
1052 // been allocated by this MemoryAllocator.
1053 V8_INLINE bool IsOutsideAllocatedSpace(const void* address) const {
1054 return address < lowest_ever_allocated_ ||
1055 address >= highest_ever_allocated_;
1059 // Reports statistic info of the space.
1060 void ReportStatistics();
1063 // Returns a MemoryChunk in which the memory region from commit_area_size to
1064 // reserve_area_size of the chunk area is reserved but not committed, it
1065 // could be committed later by calling MemoryChunk::CommitArea.
1066 MemoryChunk* AllocateChunk(intptr_t reserve_area_size,
1067 intptr_t commit_area_size,
1068 Executability executable, Space* space);
1070 Address ReserveAlignedMemory(size_t requested, size_t alignment,
1071 base::VirtualMemory* controller);
1072 Address AllocateAlignedMemory(size_t reserve_size, size_t commit_size,
1073 size_t alignment, Executability executable,
1074 base::VirtualMemory* controller);
1076 bool CommitMemory(Address addr, size_t size, Executability executable);
1078 void FreeMemory(base::VirtualMemory* reservation, Executability executable);
1079 void FreeMemory(Address addr, size_t size, Executability executable);
1081 // Commit a contiguous block of memory from the initial chunk. Assumes that
1082 // the address is not NULL, the size is greater than zero, and that the
1083 // block is contained in the initial chunk. Returns true if it succeeded
1084 // and false otherwise.
1085 bool CommitBlock(Address start, size_t size, Executability executable);
1087 // Uncommit a contiguous block of memory [start..(start+size)[.
1088 // start is not NULL, the size is greater than zero, and the
1089 // block is contained in the initial chunk. Returns true if it succeeded
1090 // and false otherwise.
1091 bool UncommitBlock(Address start, size_t size);
1093 // Zaps a contiguous block of memory [start..(start+size)[ thus
1094 // filling it up with a recognizable non-NULL bit pattern.
1095 void ZapBlock(Address start, size_t size);
1097 void PerformAllocationCallback(ObjectSpace space, AllocationAction action,
1100 void AddMemoryAllocationCallback(MemoryAllocationCallback callback,
1101 ObjectSpace space, AllocationAction action);
1103 void RemoveMemoryAllocationCallback(MemoryAllocationCallback callback);
1105 bool MemoryAllocationCallbackRegistered(MemoryAllocationCallback callback);
1107 static int CodePageGuardStartOffset();
1109 static int CodePageGuardSize();
1111 static int CodePageAreaStartOffset();
1113 static int CodePageAreaEndOffset();
1115 static int CodePageAreaSize() {
1116 return CodePageAreaEndOffset() - CodePageAreaStartOffset();
1119 static int PageAreaSize(AllocationSpace space) {
1120 DCHECK_NE(LO_SPACE, space);
1121 return (space == CODE_SPACE) ? CodePageAreaSize()
1122 : Page::kMaxRegularHeapObjectSize;
1125 MUST_USE_RESULT bool CommitExecutableMemory(base::VirtualMemory* vm,
1126 Address start, size_t commit_size,
1127 size_t reserved_size);
1132 // Maximum space size in bytes.
1134 // Maximum subset of capacity_ that can be executable
1135 size_t capacity_executable_;
1137 // Allocated space size in bytes.
1139 // Allocated executable space size in bytes.
1140 size_t size_executable_;
1142 // We keep the lowest and highest addresses allocated as a quick way
1143 // of determining that pointers are outside the heap. The estimate is
1144 // conservative, i.e. not all addrsses in 'allocated' space are allocated
1145 // to our heap. The range is [lowest, highest[, inclusive on the low end
1146 // and exclusive on the high end.
1147 void* lowest_ever_allocated_;
1148 void* highest_ever_allocated_;
1150 struct MemoryAllocationCallbackRegistration {
1151 MemoryAllocationCallbackRegistration(MemoryAllocationCallback callback,
1153 AllocationAction action)
1154 : callback(callback), space(space), action(action) {}
1155 MemoryAllocationCallback callback;
1157 AllocationAction action;
1160 // A List of callback that are triggered when memory is allocated or free'd
1161 List<MemoryAllocationCallbackRegistration> memory_allocation_callbacks_;
1163 // Initializes pages in a chunk. Returns the first page address.
1164 // This function and GetChunkId() are provided for the mark-compact
1165 // collector to rebuild page headers in the from space, which is
1166 // used as a marking stack and its page headers are destroyed.
1167 Page* InitializePagesInChunk(int chunk_id, int pages_in_chunk,
1170 void UpdateAllocatedSpaceLimits(void* low, void* high) {
1171 lowest_ever_allocated_ = Min(lowest_ever_allocated_, low);
1172 highest_ever_allocated_ = Max(highest_ever_allocated_, high);
1175 DISALLOW_IMPLICIT_CONSTRUCTORS(MemoryAllocator);
1179 // -----------------------------------------------------------------------------
1180 // Interface for heap object iterator to be implemented by all object space
1181 // object iterators.
1183 // NOTE: The space specific object iterators also implements the own next()
1184 // method which is used to avoid using virtual functions
1185 // iterating a specific space.
1187 class ObjectIterator : public Malloced {
1189 virtual ~ObjectIterator() {}
1191 virtual HeapObject* next_object() = 0;
1195 // -----------------------------------------------------------------------------
1196 // Heap object iterator in new/old/map spaces.
1198 // A HeapObjectIterator iterates objects from the bottom of the given space
1199 // to its top or from the bottom of the given page to its top.
1201 // If objects are allocated in the page during iteration the iterator may
1202 // or may not iterate over those objects. The caller must create a new
1203 // iterator in order to be sure to visit these new objects.
1204 class HeapObjectIterator : public ObjectIterator {
1206 // Creates a new object iterator in a given space.
1207 // If the size function is not given, the iterator calls the default
1209 explicit HeapObjectIterator(PagedSpace* space);
1210 HeapObjectIterator(PagedSpace* space, HeapObjectCallback size_func);
1211 HeapObjectIterator(Page* page, HeapObjectCallback size_func);
1213 // Advance to the next object, skipping free spaces and other fillers and
1214 // skipping the special garbage section of which there is one per space.
1215 // Returns NULL when the iteration has ended.
1216 inline HeapObject* Next() {
1218 HeapObject* next_obj = FromCurrentPage();
1219 if (next_obj != NULL) return next_obj;
1220 } while (AdvanceToNextPage());
1224 virtual HeapObject* next_object() { return Next(); }
1227 enum PageMode { kOnePageOnly, kAllPagesInSpace };
1229 Address cur_addr_; // Current iteration point.
1230 Address cur_end_; // End iteration point.
1231 HeapObjectCallback size_func_; // Size function or NULL.
1233 PageMode page_mode_;
1235 // Fast (inlined) path of next().
1236 inline HeapObject* FromCurrentPage();
1238 // Slow path of next(), goes into the next page. Returns false if the
1239 // iteration has ended.
1240 bool AdvanceToNextPage();
1242 // Initializes fields.
1243 inline void Initialize(PagedSpace* owner, Address start, Address end,
1244 PageMode mode, HeapObjectCallback size_func);
1248 // -----------------------------------------------------------------------------
1249 // A PageIterator iterates the pages in a paged space.
1251 class PageIterator BASE_EMBEDDED {
1253 explicit inline PageIterator(PagedSpace* space);
1255 inline bool has_next();
1256 inline Page* next();
1260 Page* prev_page_; // Previous page returned.
1261 // Next page that will be returned. Cached here so that we can use this
1262 // iterator for operations that deallocate pages.
1267 // -----------------------------------------------------------------------------
1268 // A space has a circular list of pages. The next page can be accessed via
1269 // Page::next_page() call.
1271 // An abstraction of allocation and relocation pointers in a page-structured
1273 class AllocationInfo {
1275 AllocationInfo() : top_(NULL), limit_(NULL) {}
1277 INLINE(void set_top(Address top)) {
1278 SLOW_DCHECK(top == NULL ||
1279 (reinterpret_cast<intptr_t>(top) & HeapObjectTagMask()) == 0);
1283 INLINE(Address top()) const {
1284 SLOW_DCHECK(top_ == NULL ||
1285 (reinterpret_cast<intptr_t>(top_) & HeapObjectTagMask()) == 0);
1289 Address* top_address() { return &top_; }
1291 INLINE(void set_limit(Address limit)) {
1292 SLOW_DCHECK(limit == NULL ||
1293 (reinterpret_cast<intptr_t>(limit) & HeapObjectTagMask()) == 0);
1297 INLINE(Address limit()) const {
1298 SLOW_DCHECK(limit_ == NULL ||
1299 (reinterpret_cast<intptr_t>(limit_) & HeapObjectTagMask()) ==
1304 Address* limit_address() { return &limit_; }
1307 bool VerifyPagedAllocation() {
1308 return (Page::FromAllocationTop(top_) == Page::FromAllocationTop(limit_)) &&
1314 // Current allocation top.
1316 // Current allocation limit.
1321 // An abstraction of the accounting statistics of a page-structured space.
1322 // The 'capacity' of a space is the number of object-area bytes (i.e., not
1323 // including page bookkeeping structures) currently in the space. The 'size'
1324 // of a space is the number of allocated bytes, the 'waste' in the space is
1325 // the number of bytes that are not allocated and not available to
1326 // allocation without reorganizing the space via a GC (e.g. small blocks due
1327 // to internal fragmentation, top of page areas in map space), and the bytes
1328 // 'available' is the number of unallocated bytes that are not waste. The
1329 // capacity is the sum of size, waste, and available.
1331 // The stats are only set by functions that ensure they stay balanced. These
1332 // functions increase or decrease one of the non-capacity stats in
1333 // conjunction with capacity, or else they always balance increases and
1334 // decreases to the non-capacity stats.
1335 class AllocationStats BASE_EMBEDDED {
1337 AllocationStats() { Clear(); }
1339 // Zero out all the allocation statistics (i.e., no capacity).
1347 void ClearSizeWaste() {
1352 // Reset the allocation statistics (i.e., available = capacity with no
1353 // wasted or allocated bytes).
1359 // Accessors for the allocation statistics.
1360 intptr_t Capacity() { return capacity_; }
1361 intptr_t MaxCapacity() { return max_capacity_; }
1362 intptr_t Size() { return size_; }
1363 intptr_t Waste() { return waste_; }
1365 // Grow the space by adding available bytes. They are initially marked as
1366 // being in use (part of the size), but will normally be immediately freed,
1367 // putting them on the free list and removing them from size_.
1368 void ExpandSpace(int size_in_bytes) {
1369 capacity_ += size_in_bytes;
1370 size_ += size_in_bytes;
1371 if (capacity_ > max_capacity_) {
1372 max_capacity_ = capacity_;
1377 // Shrink the space by removing available bytes. Since shrinking is done
1378 // during sweeping, bytes have been marked as being in use (part of the size)
1379 // and are hereby freed.
1380 void ShrinkSpace(int size_in_bytes) {
1381 capacity_ -= size_in_bytes;
1382 size_ -= size_in_bytes;
1386 // Allocate from available bytes (available -> size).
1387 void AllocateBytes(intptr_t size_in_bytes) {
1388 size_ += size_in_bytes;
1392 // Free allocated bytes, making them available (size -> available).
1393 void DeallocateBytes(intptr_t size_in_bytes) {
1394 size_ -= size_in_bytes;
1398 // Waste free bytes (available -> waste).
1399 void WasteBytes(int size_in_bytes) {
1400 DCHECK(size_in_bytes >= 0);
1401 waste_ += size_in_bytes;
1406 intptr_t max_capacity_;
1412 // -----------------------------------------------------------------------------
1413 // Free lists for old object spaces
1415 // Free-list nodes are free blocks in the heap. They look like heap objects
1416 // (free-list node pointers have the heap object tag, and they have a map like
1417 // a heap object). They have a size and a next pointer. The next pointer is
1418 // the raw address of the next free list node (or NULL).
1419 class FreeListNode : public HeapObject {
1421 // Obtain a free-list node from a raw address. This is not a cast because
1422 // it does not check nor require that the first word at the address is a map
1424 static FreeListNode* FromAddress(Address address) {
1425 return reinterpret_cast<FreeListNode*>(HeapObject::FromAddress(address));
1428 static inline bool IsFreeListNode(HeapObject* object);
1430 // Set the size in bytes, which can be read with HeapObject::Size(). This
1431 // function also writes a map to the first word of the block so that it
1432 // looks like a heap object to the garbage collector and heap iteration
1434 void set_size(Heap* heap, int size_in_bytes);
1436 // Accessors for the next field.
1437 inline FreeListNode* next();
1438 inline FreeListNode** next_address();
1439 inline void set_next(FreeListNode* next);
1443 static inline FreeListNode* cast(Object* object) {
1444 return reinterpret_cast<FreeListNode*>(object);
1448 static const int kNextOffset = POINTER_SIZE_ALIGN(FreeSpace::kHeaderSize);
1450 DISALLOW_IMPLICIT_CONSTRUCTORS(FreeListNode);
1454 // The free list category holds a pointer to the top element and a pointer to
1455 // the end element of the linked list of free memory blocks.
1456 class FreeListCategory {
1458 FreeListCategory() : top_(0), end_(NULL), available_(0) {}
1460 intptr_t Concatenate(FreeListCategory* category);
1464 void Free(FreeListNode* node, int size_in_bytes);
1466 FreeListNode* PickNodeFromList(int* node_size);
1467 FreeListNode* PickNodeFromList(int size_in_bytes, int* node_size);
1469 intptr_t EvictFreeListItemsInList(Page* p);
1470 bool ContainsPageFreeListItemsInList(Page* p);
1472 void RepairFreeList(Heap* heap);
1474 FreeListNode* top() const {
1475 return reinterpret_cast<FreeListNode*>(base::NoBarrier_Load(&top_));
1478 void set_top(FreeListNode* top) {
1479 base::NoBarrier_Store(&top_, reinterpret_cast<base::AtomicWord>(top));
1482 FreeListNode** GetEndAddress() { return &end_; }
1483 FreeListNode* end() const { return end_; }
1484 void set_end(FreeListNode* end) { end_ = end; }
1486 int* GetAvailableAddress() { return &available_; }
1487 int available() const { return available_; }
1488 void set_available(int available) { available_ = available; }
1490 base::Mutex* mutex() { return &mutex_; }
1492 bool IsEmpty() { return top() == 0; }
1495 intptr_t SumFreeList();
1496 int FreeListLength();
1500 // top_ points to the top FreeListNode* in the free list category.
1501 base::AtomicWord top_;
1505 // Total available bytes in all blocks of this free list category.
1510 // The free list for the old space. The free list is organized in such a way
1511 // as to encourage objects allocated around the same time to be near each
1512 // other. The normal way to allocate is intended to be by bumping a 'top'
1513 // pointer until it hits a 'limit' pointer. When the limit is hit we need to
1514 // find a new space to allocate from. This is done with the free list, which
1515 // is divided up into rough categories to cut down on waste. Having finer
1516 // categories would scatter allocation more.
1518 // The old space free list is organized in categories.
1519 // 1-31 words: Such small free areas are discarded for efficiency reasons.
1520 // They can be reclaimed by the compactor. However the distance between top
1521 // and limit may be this small.
1522 // 32-255 words: There is a list of spaces this large. It is used for top and
1523 // limit when the object we need to allocate is 1-31 words in size. These
1524 // spaces are called small.
1525 // 256-2047 words: There is a list of spaces this large. It is used for top and
1526 // limit when the object we need to allocate is 32-255 words in size. These
1527 // spaces are called medium.
1528 // 1048-16383 words: There is a list of spaces this large. It is used for top
1529 // and limit when the object we need to allocate is 256-2047 words in size.
1530 // These spaces are call large.
1531 // At least 16384 words. This list is for objects of 2048 words or larger.
1532 // Empty pages are added to this list. These spaces are called huge.
1535 explicit FreeList(PagedSpace* owner);
1537 intptr_t Concatenate(FreeList* free_list);
1539 // Clear the free list.
1542 // Return the number of bytes available on the free list.
1543 intptr_t available() {
1544 return small_list_.available() + medium_list_.available() +
1545 large_list_.available() + huge_list_.available();
1548 // Place a node on the free list. The block of size 'size_in_bytes'
1549 // starting at 'start' is placed on the free list. The return value is the
1550 // number of bytes that have been lost due to internal fragmentation by
1551 // freeing the block. Bookkeeping information will be written to the block,
1552 // i.e., its contents will be destroyed. The start address should be word
1553 // aligned, and the size should be a non-zero multiple of the word size.
1554 int Free(Address start, int size_in_bytes);
1556 // This method returns how much memory can be allocated after freeing
1557 // maximum_freed memory.
1558 static inline int GuaranteedAllocatable(int maximum_freed) {
1559 if (maximum_freed < kSmallListMin) {
1561 } else if (maximum_freed <= kSmallListMax) {
1562 return kSmallAllocationMax;
1563 } else if (maximum_freed <= kMediumListMax) {
1564 return kMediumAllocationMax;
1565 } else if (maximum_freed <= kLargeListMax) {
1566 return kLargeAllocationMax;
1568 return maximum_freed;
1571 // Allocate a block of size 'size_in_bytes' from the free list. The block
1572 // is unitialized. A failure is returned if no block is available. The
1573 // number of bytes lost to fragmentation is returned in the output parameter
1574 // 'wasted_bytes'. The size should be a non-zero multiple of the word size.
1575 MUST_USE_RESULT HeapObject* Allocate(int size_in_bytes);
1578 return small_list_.IsEmpty() && medium_list_.IsEmpty() &&
1579 large_list_.IsEmpty() && huge_list_.IsEmpty();
1584 intptr_t SumFreeLists();
1588 // Used after booting the VM.
1589 void RepairLists(Heap* heap);
1591 intptr_t EvictFreeListItems(Page* p);
1592 bool ContainsPageFreeListItems(Page* p);
1594 FreeListCategory* small_list() { return &small_list_; }
1595 FreeListCategory* medium_list() { return &medium_list_; }
1596 FreeListCategory* large_list() { return &large_list_; }
1597 FreeListCategory* huge_list() { return &huge_list_; }
1600 // The size range of blocks, in bytes.
1601 static const int kMinBlockSize = 3 * kPointerSize;
1602 static const int kMaxBlockSize = Page::kMaxRegularHeapObjectSize;
1604 FreeListNode* FindNodeFor(int size_in_bytes, int* node_size);
1609 static const int kSmallListMin = 0x20 * kPointerSize;
1610 static const int kSmallListMax = 0xff * kPointerSize;
1611 static const int kMediumListMax = 0x7ff * kPointerSize;
1612 static const int kLargeListMax = 0x3fff * kPointerSize;
1613 static const int kSmallAllocationMax = kSmallListMin - kPointerSize;
1614 static const int kMediumAllocationMax = kSmallListMax;
1615 static const int kLargeAllocationMax = kMediumListMax;
1616 FreeListCategory small_list_;
1617 FreeListCategory medium_list_;
1618 FreeListCategory large_list_;
1619 FreeListCategory huge_list_;
1621 DISALLOW_IMPLICIT_CONSTRUCTORS(FreeList);
1625 class AllocationResult {
1627 // Implicit constructor from Object*.
1628 AllocationResult(Object* object) // NOLINT
1630 // AllocationResults can't return Smis, which are used to represent
1631 // failure and the space to retry in.
1632 CHECK(!object->IsSmi());
1635 AllocationResult() : object_(Smi::FromInt(NEW_SPACE)) {}
1637 static inline AllocationResult Retry(AllocationSpace space = NEW_SPACE) {
1638 return AllocationResult(space);
1641 inline bool IsRetry() { return object_->IsSmi(); }
1643 template <typename T>
1645 if (IsRetry()) return false;
1646 *obj = T::cast(object_);
1650 Object* ToObjectChecked() {
1655 AllocationSpace RetrySpace() {
1657 return static_cast<AllocationSpace>(Smi::cast(object_)->value());
1661 explicit AllocationResult(AllocationSpace space)
1662 : object_(Smi::FromInt(static_cast<int>(space))) {}
1668 STATIC_ASSERT(sizeof(AllocationResult) == kPointerSize);
1671 class PagedSpace : public Space {
1673 // Creates a space with a maximum capacity, and an id.
1674 PagedSpace(Heap* heap, intptr_t max_capacity, AllocationSpace id,
1675 Executability executable);
1677 virtual ~PagedSpace() {}
1679 // Set up the space using the given address range of virtual memory (from
1680 // the memory allocator's initial chunk) if possible. If the block of
1681 // addresses is not big enough to contain a single page-aligned page, a
1682 // fresh chunk will be allocated.
1685 // Returns true if the space has been successfully set up and not
1686 // subsequently torn down.
1687 bool HasBeenSetUp();
1689 // Cleans up the space, frees all pages in this space except those belonging
1690 // to the initial chunk, uncommits addresses in the initial chunk.
1693 // Checks whether an object/address is in this space.
1694 inline bool Contains(Address a);
1695 bool Contains(HeapObject* o) { return Contains(o->address()); }
1697 // Given an address occupied by a live object, return that object if it is
1698 // in this space, or a Smi if it is not. The implementation iterates over
1699 // objects in the page containing the address, the cost is linear in the
1700 // number of objects in the page. It may be slow.
1701 Object* FindObject(Address addr);
1703 // During boot the free_space_map is created, and afterwards we may need
1704 // to write it into the free list nodes that were already created.
1705 void RepairFreeListsAfterBoot();
1707 // Prepares for a mark-compact GC.
1708 void PrepareForMarkCompact();
1710 // Current capacity without growing (Size() + Available()).
1711 intptr_t Capacity() { return accounting_stats_.Capacity(); }
1713 // Total amount of memory committed for this space. For paged
1714 // spaces this equals the capacity.
1715 intptr_t CommittedMemory() { return Capacity(); }
1717 // The maximum amount of memory ever committed for this space.
1718 intptr_t MaximumCommittedMemory() { return accounting_stats_.MaxCapacity(); }
1720 // Approximate amount of physical memory committed for this space.
1721 size_t CommittedPhysicalMemory();
1725 return small_size_ + medium_size_ + large_size_ + huge_size_;
1728 intptr_t small_size_;
1729 intptr_t medium_size_;
1730 intptr_t large_size_;
1731 intptr_t huge_size_;
1734 void ObtainFreeListStatistics(Page* p, SizeStats* sizes);
1735 void ResetFreeListStatistics();
1737 // Sets the capacity, the available space and the wasted space to zero.
1738 // The stats are rebuilt during sweeping by adding each page to the
1739 // capacity and the size when it is encountered. As free spaces are
1740 // discovered during the sweeping they are subtracted from the size and added
1741 // to the available and wasted totals.
1743 accounting_stats_.ClearSizeWaste();
1744 ResetFreeListStatistics();
1747 // Increases the number of available bytes of that space.
1748 void AddToAccountingStats(intptr_t bytes) {
1749 accounting_stats_.DeallocateBytes(bytes);
1752 // Available bytes without growing. These are the bytes on the free list.
1753 // The bytes in the linear allocation area are not included in this total
1754 // because updating the stats would slow down allocation. New pages are
1755 // immediately added to the free list so they show up here.
1756 intptr_t Available() { return free_list_.available(); }
1758 // Allocated bytes in this space. Garbage bytes that were not found due to
1759 // concurrent sweeping are counted as being allocated! The bytes in the
1760 // current linear allocation area (between top and limit) are also counted
1762 virtual intptr_t Size() { return accounting_stats_.Size(); }
1764 // As size, but the bytes in lazily swept pages are estimated and the bytes
1765 // in the current linear allocation area are not included.
1766 virtual intptr_t SizeOfObjects();
1768 // Wasted bytes in this space. These are just the bytes that were thrown away
1769 // due to being too small to use for allocation. They do not include the
1770 // free bytes that were not found at all due to lazy sweeping.
1771 virtual intptr_t Waste() { return accounting_stats_.Waste(); }
1773 // Returns the allocation pointer in this space.
1774 Address top() { return allocation_info_.top(); }
1775 Address limit() { return allocation_info_.limit(); }
1777 // The allocation top address.
1778 Address* allocation_top_address() { return allocation_info_.top_address(); }
1780 // The allocation limit address.
1781 Address* allocation_limit_address() {
1782 return allocation_info_.limit_address();
1785 // Allocate the requested number of bytes in the space if possible, return a
1786 // failure object if not.
1787 MUST_USE_RESULT inline AllocationResult AllocateRaw(int size_in_bytes);
1789 // Give a block of memory to the space's free list. It might be added to
1790 // the free list or accounted as waste.
1791 // If add_to_freelist is false then just accounting stats are updated and
1792 // no attempt to add area to free list is made.
1793 int Free(Address start, int size_in_bytes) {
1794 int wasted = free_list_.Free(start, size_in_bytes);
1795 accounting_stats_.DeallocateBytes(size_in_bytes);
1796 accounting_stats_.WasteBytes(wasted);
1797 return size_in_bytes - wasted;
1800 void ResetFreeList() { free_list_.Reset(); }
1802 // Set space allocation info.
1803 void SetTopAndLimit(Address top, Address limit) {
1804 DCHECK(top == limit ||
1805 Page::FromAddress(top) == Page::FromAddress(limit - 1));
1806 MemoryChunk::UpdateHighWaterMark(allocation_info_.top());
1807 allocation_info_.set_top(top);
1808 allocation_info_.set_limit(limit);
1811 // Empty space allocation info, returning unused area to free list.
1812 void EmptyAllocationInfo() {
1813 // Mark the old linear allocation area with a free space map so it can be
1814 // skipped when scanning the heap.
1815 int old_linear_size = static_cast<int>(limit() - top());
1816 Free(top(), old_linear_size);
1817 SetTopAndLimit(NULL, NULL);
1820 void Allocate(int bytes) { accounting_stats_.AllocateBytes(bytes); }
1822 void IncreaseCapacity(int size);
1824 // Releases an unused page and shrinks the space.
1825 void ReleasePage(Page* page);
1827 // The dummy page that anchors the linked list of pages.
1828 Page* anchor() { return &anchor_; }
1831 // Verify integrity of this space.
1832 virtual void Verify(ObjectVisitor* visitor);
1834 // Overridden by subclasses to verify space-specific object
1835 // properties (e.g., only maps or free-list nodes are in map space).
1836 virtual void VerifyObject(HeapObject* obj) {}
1840 // Print meta info and objects in this space.
1841 virtual void Print();
1843 // Reports statistics for the space
1844 void ReportStatistics();
1846 // Report code object related statistics
1847 void CollectCodeStatistics();
1848 static void ReportCodeStatistics(Isolate* isolate);
1849 static void ResetCodeStatistics(Isolate* isolate);
1852 // Evacuation candidates are swept by evacuator. Needs to return a valid
1853 // result before _and_ after evacuation has finished.
1854 static bool ShouldBeSweptBySweeperThreads(Page* p) {
1855 return !p->IsEvacuationCandidate() &&
1856 !p->IsFlagSet(Page::RESCAN_ON_EVACUATION) && !p->WasSwept();
1859 void IncrementUnsweptFreeBytes(intptr_t by) { unswept_free_bytes_ += by; }
1861 void IncreaseUnsweptFreeBytes(Page* p) {
1862 DCHECK(ShouldBeSweptBySweeperThreads(p));
1863 unswept_free_bytes_ += (p->area_size() - p->LiveBytes());
1866 void DecrementUnsweptFreeBytes(intptr_t by) { unswept_free_bytes_ -= by; }
1868 void DecreaseUnsweptFreeBytes(Page* p) {
1869 DCHECK(ShouldBeSweptBySweeperThreads(p));
1870 unswept_free_bytes_ -= (p->area_size() - p->LiveBytes());
1873 void ResetUnsweptFreeBytes() { unswept_free_bytes_ = 0; }
1875 // This function tries to steal size_in_bytes memory from the sweeper threads
1876 // free-lists. If it does not succeed stealing enough memory, it will wait
1877 // for the sweeper threads to finish sweeping.
1878 // It returns true when sweeping is completed and false otherwise.
1879 bool EnsureSweeperProgress(intptr_t size_in_bytes);
1881 void set_end_of_unswept_pages(Page* page) { end_of_unswept_pages_ = page; }
1883 Page* end_of_unswept_pages() { return end_of_unswept_pages_; }
1885 Page* FirstPage() { return anchor_.next_page(); }
1886 Page* LastPage() { return anchor_.prev_page(); }
1888 void EvictEvacuationCandidatesFromFreeLists();
1892 // Returns the number of total pages in this space.
1893 int CountTotalPages();
1895 // Return size of allocatable area on a page in this space.
1896 inline int AreaSize() { return area_size_; }
1898 void CreateEmergencyMemory();
1899 void FreeEmergencyMemory();
1900 void UseEmergencyMemory();
1902 bool HasEmergencyMemory() { return emergency_memory_ != NULL; }
1905 FreeList* free_list() { return &free_list_; }
1909 // Maximum capacity of this space.
1910 intptr_t max_capacity_;
1912 intptr_t SizeOfFirstPage();
1914 // Accounting information for this space.
1915 AllocationStats accounting_stats_;
1917 // The dummy page that anchors the double linked list of pages.
1920 // The space's free list.
1921 FreeList free_list_;
1923 // Normal allocation information.
1924 AllocationInfo allocation_info_;
1926 // The number of free bytes which could be reclaimed by advancing the
1927 // concurrent sweeper threads.
1928 intptr_t unswept_free_bytes_;
1930 // The sweeper threads iterate over the list of pointer and data space pages
1931 // and sweep these pages concurrently. They will stop sweeping after the
1932 // end_of_unswept_pages_ page.
1933 Page* end_of_unswept_pages_;
1935 // Emergency memory is the memory of a full page for a given space, allocated
1936 // conservatively before evacuating a page. If compaction fails due to out
1937 // of memory error the emergency memory can be used to complete compaction.
1938 // If not used, the emergency memory is released after compaction.
1939 MemoryChunk* emergency_memory_;
1941 // Expands the space by allocating a fixed number of pages. Returns false if
1942 // it cannot allocate requested number of pages from OS, or if the hard heap
1943 // size limit has been hit.
1946 // Generic fast case allocation function that tries linear allocation at the
1947 // address denoted by top in allocation_info_.
1948 inline HeapObject* AllocateLinearly(int size_in_bytes);
1950 // If sweeping is still in progress try to sweep unswept pages. If that is
1951 // not successful, wait for the sweeper threads and re-try free-list
1953 MUST_USE_RESULT HeapObject* WaitForSweeperThreadsAndRetryAllocation(
1956 // Slow path of AllocateRaw. This function is space-dependent.
1957 MUST_USE_RESULT HeapObject* SlowAllocateRaw(int size_in_bytes);
1959 friend class PageIterator;
1960 friend class MarkCompactCollector;
1964 class NumberAndSizeInfo BASE_EMBEDDED {
1966 NumberAndSizeInfo() : number_(0), bytes_(0) {}
1968 int number() const { return number_; }
1969 void increment_number(int num) { number_ += num; }
1971 int bytes() const { return bytes_; }
1972 void increment_bytes(int size) { bytes_ += size; }
1985 // HistogramInfo class for recording a single "bar" of a histogram. This
1986 // class is used for collecting statistics to print to the log file.
1987 class HistogramInfo : public NumberAndSizeInfo {
1989 HistogramInfo() : NumberAndSizeInfo() {}
1991 const char* name() { return name_; }
1992 void set_name(const char* name) { name_ = name; }
1999 enum SemiSpaceId { kFromSpace = 0, kToSpace = 1 };
2005 class NewSpacePage : public MemoryChunk {
2007 // GC related flags copied from from-space to to-space when
2008 // flipping semispaces.
2009 static const intptr_t kCopyOnFlipFlagsMask =
2010 (1 << MemoryChunk::POINTERS_TO_HERE_ARE_INTERESTING) |
2011 (1 << MemoryChunk::POINTERS_FROM_HERE_ARE_INTERESTING) |
2012 (1 << MemoryChunk::SCAN_ON_SCAVENGE);
2014 static const int kAreaSize = Page::kMaxRegularHeapObjectSize;
2016 inline NewSpacePage* next_page() const {
2017 return static_cast<NewSpacePage*>(next_chunk());
2020 inline void set_next_page(NewSpacePage* page) { set_next_chunk(page); }
2022 inline NewSpacePage* prev_page() const {
2023 return static_cast<NewSpacePage*>(prev_chunk());
2026 inline void set_prev_page(NewSpacePage* page) { set_prev_chunk(page); }
2028 SemiSpace* semi_space() { return reinterpret_cast<SemiSpace*>(owner()); }
2030 bool is_anchor() { return !this->InNewSpace(); }
2032 static bool IsAtStart(Address addr) {
2033 return (reinterpret_cast<intptr_t>(addr) & Page::kPageAlignmentMask) ==
2037 static bool IsAtEnd(Address addr) {
2038 return (reinterpret_cast<intptr_t>(addr) & Page::kPageAlignmentMask) == 0;
2041 Address address() { return reinterpret_cast<Address>(this); }
2043 // Finds the NewSpacePage containing the given address.
2044 static inline NewSpacePage* FromAddress(Address address_in_page) {
2045 Address page_start =
2046 reinterpret_cast<Address>(reinterpret_cast<uintptr_t>(address_in_page) &
2047 ~Page::kPageAlignmentMask);
2048 NewSpacePage* page = reinterpret_cast<NewSpacePage*>(page_start);
2052 // Find the page for a limit address. A limit address is either an address
2053 // inside a page, or the address right after the last byte of a page.
2054 static inline NewSpacePage* FromLimit(Address address_limit) {
2055 return NewSpacePage::FromAddress(address_limit - 1);
2058 // Checks if address1 and address2 are on the same new space page.
2059 static inline bool OnSamePage(Address address1, Address address2) {
2060 return NewSpacePage::FromAddress(address1) ==
2061 NewSpacePage::FromAddress(address2);
2065 // Create a NewSpacePage object that is only used as anchor
2066 // for the doubly-linked list of real pages.
2067 explicit NewSpacePage(SemiSpace* owner) { InitializeAsAnchor(owner); }
2069 static NewSpacePage* Initialize(Heap* heap, Address start,
2070 SemiSpace* semi_space);
2072 // Intialize a fake NewSpacePage used as sentinel at the ends
2073 // of a doubly-linked list of real NewSpacePages.
2074 // Only uses the prev/next links, and sets flags to not be in new-space.
2075 void InitializeAsAnchor(SemiSpace* owner);
2077 friend class SemiSpace;
2078 friend class SemiSpaceIterator;
2082 // -----------------------------------------------------------------------------
2083 // SemiSpace in young generation
2085 // A semispace is a contiguous chunk of memory holding page-like memory
2086 // chunks. The mark-compact collector uses the memory of the first page in
2087 // the from space as a marking stack when tracing live objects.
2089 class SemiSpace : public Space {
2092 SemiSpace(Heap* heap, SemiSpaceId semispace)
2093 : Space(heap, NEW_SPACE, NOT_EXECUTABLE),
2098 current_page_(NULL) {}
2100 // Sets up the semispace using the given chunk.
2101 void SetUp(Address start, int initial_capacity, int target_capacity,
2102 int maximum_capacity);
2104 // Tear down the space. Heap memory was not allocated by the space, so it
2105 // is not deallocated here.
2108 // True if the space has been set up but not torn down.
2109 bool HasBeenSetUp() { return start_ != NULL; }
2111 // Grow the semispace to the new capacity. The new capacity
2112 // requested must be larger than the current capacity and less than
2113 // the maximum capacity.
2114 bool GrowTo(int new_capacity);
2116 // Shrinks the semispace to the new capacity. The new capacity
2117 // requested must be more than the amount of used memory in the
2118 // semispace and less than the current capacity.
2119 bool ShrinkTo(int new_capacity);
2121 // Sets the total capacity. Only possible when the space is not committed.
2122 bool SetTotalCapacity(int new_capacity);
2124 // Returns the start address of the first page of the space.
2125 Address space_start() {
2126 DCHECK(anchor_.next_page() != &anchor_);
2127 return anchor_.next_page()->area_start();
2130 // Returns the start address of the current page of the space.
2131 Address page_low() { return current_page_->area_start(); }
2133 // Returns one past the end address of the space.
2134 Address space_end() { return anchor_.prev_page()->area_end(); }
2136 // Returns one past the end address of the current page of the space.
2137 Address page_high() { return current_page_->area_end(); }
2139 bool AdvancePage() {
2140 NewSpacePage* next_page = current_page_->next_page();
2141 if (next_page == anchor()) return false;
2142 current_page_ = next_page;
2146 // Resets the space to using the first page.
2149 // Age mark accessors.
2150 Address age_mark() { return age_mark_; }
2151 void set_age_mark(Address mark);
2153 // True if the address is in the address range of this semispace (not
2154 // necessarily below the allocation pointer).
2155 bool Contains(Address a) {
2156 return (reinterpret_cast<uintptr_t>(a) & address_mask_) ==
2157 reinterpret_cast<uintptr_t>(start_);
2160 // True if the object is a heap object in the address range of this
2161 // semispace (not necessarily below the allocation pointer).
2162 bool Contains(Object* o) {
2163 return (reinterpret_cast<uintptr_t>(o) & object_mask_) == object_expected_;
2166 // If we don't have these here then SemiSpace will be abstract. However
2167 // they should never be called.
2168 virtual intptr_t Size() {
2173 bool is_committed() { return committed_; }
2177 NewSpacePage* first_page() { return anchor_.next_page(); }
2178 NewSpacePage* current_page() { return current_page_; }
2181 virtual void Verify();
2185 virtual void Print();
2186 // Validate a range of of addresses in a SemiSpace.
2187 // The "from" address must be on a page prior to the "to" address,
2188 // in the linked page order, or it must be earlier on the same page.
2189 static void AssertValidRange(Address from, Address to);
2192 inline static void AssertValidRange(Address from, Address to) {}
2195 // Returns the current total capacity of the semispace.
2196 int TotalCapacity() { return total_capacity_; }
2198 // Returns the target for total capacity of the semispace.
2199 int TargetCapacity() { return target_capacity_; }
2201 // Returns the maximum total capacity of the semispace.
2202 int MaximumTotalCapacity() { return maximum_total_capacity_; }
2204 // Returns the initial capacity of the semispace.
2205 int InitialTotalCapacity() { return initial_total_capacity_; }
2207 SemiSpaceId id() { return id_; }
2209 static void Swap(SemiSpace* from, SemiSpace* to);
2211 // Returns the maximum amount of memory ever committed by the semi space.
2212 size_t MaximumCommittedMemory() { return maximum_committed_; }
2214 // Approximate amount of physical memory committed for this space.
2215 size_t CommittedPhysicalMemory();
2218 // Flips the semispace between being from-space and to-space.
2219 // Copies the flags into the masked positions on all pages in the space.
2220 void FlipPages(intptr_t flags, intptr_t flag_mask);
2222 // Updates Capacity and MaximumCommitted based on new capacity.
2223 void SetCapacity(int new_capacity);
2225 NewSpacePage* anchor() { return &anchor_; }
2227 // The current and maximum total capacity of the space.
2228 int total_capacity_;
2229 int target_capacity_;
2230 int maximum_total_capacity_;
2231 int initial_total_capacity_;
2233 intptr_t maximum_committed_;
2235 // The start address of the space.
2237 // Used to govern object promotion during mark-compact collection.
2240 // Masks and comparison values to test for containment in this semispace.
2241 uintptr_t address_mask_;
2242 uintptr_t object_mask_;
2243 uintptr_t object_expected_;
2248 NewSpacePage anchor_;
2249 NewSpacePage* current_page_;
2251 friend class SemiSpaceIterator;
2252 friend class NewSpacePageIterator;
2255 TRACK_MEMORY("SemiSpace")
2259 // A SemiSpaceIterator is an ObjectIterator that iterates over the active
2260 // semispace of the heap's new space. It iterates over the objects in the
2261 // semispace from a given start address (defaulting to the bottom of the
2262 // semispace) to the top of the semispace. New objects allocated after the
2263 // iterator is created are not iterated.
2264 class SemiSpaceIterator : public ObjectIterator {
2266 // Create an iterator over the objects in the given space. If no start
2267 // address is given, the iterator starts from the bottom of the space. If
2268 // no size function is given, the iterator calls Object::Size().
2270 // Iterate over all of allocated to-space.
2271 explicit SemiSpaceIterator(NewSpace* space);
2272 // Iterate over all of allocated to-space, with a custome size function.
2273 SemiSpaceIterator(NewSpace* space, HeapObjectCallback size_func);
2274 // Iterate over part of allocated to-space, from start to the end
2276 SemiSpaceIterator(NewSpace* space, Address start);
2277 // Iterate from one address to another in the same semi-space.
2278 SemiSpaceIterator(Address from, Address to);
2280 HeapObject* Next() {
2281 if (current_ == limit_) return NULL;
2282 if (NewSpacePage::IsAtEnd(current_)) {
2283 NewSpacePage* page = NewSpacePage::FromLimit(current_);
2284 page = page->next_page();
2285 DCHECK(!page->is_anchor());
2286 current_ = page->area_start();
2287 if (current_ == limit_) return NULL;
2290 HeapObject* object = HeapObject::FromAddress(current_);
2291 int size = (size_func_ == NULL) ? object->Size() : size_func_(object);
2297 // Implementation of the ObjectIterator functions.
2298 virtual HeapObject* next_object() { return Next(); }
2301 void Initialize(Address start, Address end, HeapObjectCallback size_func);
2303 // The current iteration point.
2305 // The end of iteration.
2307 // The callback function.
2308 HeapObjectCallback size_func_;
2312 // -----------------------------------------------------------------------------
2313 // A PageIterator iterates the pages in a semi-space.
2314 class NewSpacePageIterator BASE_EMBEDDED {
2316 // Make an iterator that runs over all pages in to-space.
2317 explicit inline NewSpacePageIterator(NewSpace* space);
2319 // Make an iterator that runs over all pages in the given semispace,
2320 // even those not used in allocation.
2321 explicit inline NewSpacePageIterator(SemiSpace* space);
2323 // Make iterator that iterates from the page containing start
2324 // to the page that contains limit in the same semispace.
2325 inline NewSpacePageIterator(Address start, Address limit);
2327 inline bool has_next();
2328 inline NewSpacePage* next();
2331 NewSpacePage* prev_page_; // Previous page returned.
2332 // Next page that will be returned. Cached here so that we can use this
2333 // iterator for operations that deallocate pages.
2334 NewSpacePage* next_page_;
2335 // Last page returned.
2336 NewSpacePage* last_page_;
2340 // -----------------------------------------------------------------------------
2341 // The young generation space.
2343 // The new space consists of a contiguous pair of semispaces. It simply
2344 // forwards most functions to the appropriate semispace.
2346 class NewSpace : public Space {
2349 explicit NewSpace(Heap* heap)
2350 : Space(heap, NEW_SPACE, NOT_EXECUTABLE),
2351 to_space_(heap, kToSpace),
2352 from_space_(heap, kFromSpace),
2354 inline_allocation_limit_step_(0) {}
2356 // Sets up the new space using the given chunk.
2357 bool SetUp(int reserved_semispace_size_, int max_semi_space_size);
2359 // Tears down the space. Heap memory was not allocated by the space, so it
2360 // is not deallocated here.
2363 // True if the space has been set up but not torn down.
2364 bool HasBeenSetUp() {
2365 return to_space_.HasBeenSetUp() && from_space_.HasBeenSetUp();
2368 // Flip the pair of spaces.
2371 // Grow the capacity of the semispaces. Assumes that they are not at
2372 // their maximum capacity.
2375 // Grow the capacity of the semispaces by one page.
2378 // Shrink the capacity of the semispaces.
2381 // True if the address or object lies in the address range of either
2382 // semispace (not necessarily below the allocation pointer).
2383 bool Contains(Address a) {
2384 return (reinterpret_cast<uintptr_t>(a) & address_mask_) ==
2385 reinterpret_cast<uintptr_t>(start_);
2388 bool Contains(Object* o) {
2389 Address a = reinterpret_cast<Address>(o);
2390 return (reinterpret_cast<uintptr_t>(a) & object_mask_) == object_expected_;
2393 // Return the allocated bytes in the active semispace.
2394 virtual intptr_t Size() {
2395 return pages_used_ * NewSpacePage::kAreaSize +
2396 static_cast<int>(top() - to_space_.page_low());
2399 // The same, but returning an int. We have to have the one that returns
2400 // intptr_t because it is inherited, but if we know we are dealing with the
2401 // new space, which can't get as big as the other spaces then this is useful:
2402 int SizeAsInt() { return static_cast<int>(Size()); }
2404 // Return the allocatable capacity of a semispace.
2405 intptr_t Capacity() {
2406 SLOW_DCHECK(to_space_.TotalCapacity() == from_space_.TotalCapacity());
2407 return (to_space_.TotalCapacity() / Page::kPageSize) *
2408 NewSpacePage::kAreaSize;
2411 // Return the current size of a semispace, allocatable and non-allocatable
2413 intptr_t TotalCapacity() {
2414 DCHECK(to_space_.TotalCapacity() == from_space_.TotalCapacity());
2415 return to_space_.TotalCapacity();
2418 // Return the total amount of memory committed for new space.
2419 intptr_t CommittedMemory() {
2420 if (from_space_.is_committed()) return 2 * Capacity();
2421 return TotalCapacity();
2424 // Return the total amount of memory committed for new space.
2425 intptr_t MaximumCommittedMemory() {
2426 return to_space_.MaximumCommittedMemory() +
2427 from_space_.MaximumCommittedMemory();
2430 // Approximate amount of physical memory committed for this space.
2431 size_t CommittedPhysicalMemory();
2433 // Return the available bytes without growing.
2434 intptr_t Available() { return Capacity() - Size(); }
2436 // Return the maximum capacity of a semispace.
2437 int MaximumCapacity() {
2438 DCHECK(to_space_.MaximumTotalCapacity() ==
2439 from_space_.MaximumTotalCapacity());
2440 return to_space_.MaximumTotalCapacity();
2443 bool IsAtMaximumCapacity() { return TotalCapacity() == MaximumCapacity(); }
2445 // Returns the initial capacity of a semispace.
2446 int InitialTotalCapacity() {
2447 DCHECK(to_space_.InitialTotalCapacity() ==
2448 from_space_.InitialTotalCapacity());
2449 return to_space_.InitialTotalCapacity();
2452 // Return the address of the allocation pointer in the active semispace.
2454 DCHECK(to_space_.current_page()->ContainsLimit(allocation_info_.top()));
2455 return allocation_info_.top();
2458 void set_top(Address top) {
2459 DCHECK(to_space_.current_page()->ContainsLimit(top));
2460 allocation_info_.set_top(top);
2463 // Return the address of the allocation pointer limit in the active semispace.
2465 DCHECK(to_space_.current_page()->ContainsLimit(allocation_info_.limit()));
2466 return allocation_info_.limit();
2469 // Return the address of the first object in the active semispace.
2470 Address bottom() { return to_space_.space_start(); }
2472 // Get the age mark of the inactive semispace.
2473 Address age_mark() { return from_space_.age_mark(); }
2474 // Set the age mark in the active semispace.
2475 void set_age_mark(Address mark) { to_space_.set_age_mark(mark); }
2477 // The start address of the space and a bit mask. Anding an address in the
2478 // new space with the mask will result in the start address.
2479 Address start() { return start_; }
2480 uintptr_t mask() { return address_mask_; }
2482 INLINE(uint32_t AddressToMarkbitIndex(Address addr)) {
2483 DCHECK(Contains(addr));
2484 DCHECK(IsAligned(OffsetFrom(addr), kPointerSize) ||
2485 IsAligned(OffsetFrom(addr) - 1, kPointerSize));
2486 return static_cast<uint32_t>(addr - start_) >> kPointerSizeLog2;
2489 INLINE(Address MarkbitIndexToAddress(uint32_t index)) {
2490 return reinterpret_cast<Address>(index << kPointerSizeLog2);
2493 // The allocation top and limit address.
2494 Address* allocation_top_address() { return allocation_info_.top_address(); }
2496 // The allocation limit address.
2497 Address* allocation_limit_address() {
2498 return allocation_info_.limit_address();
2501 MUST_USE_RESULT INLINE(AllocationResult AllocateRaw(int size_in_bytes));
2503 // Reset the allocation pointer to the beginning of the active semispace.
2504 void ResetAllocationInfo();
2506 void UpdateInlineAllocationLimit(int size_in_bytes);
2507 void LowerInlineAllocationLimit(intptr_t step) {
2508 inline_allocation_limit_step_ = step;
2509 UpdateInlineAllocationLimit(0);
2510 top_on_previous_step_ = allocation_info_.top();
2513 // Get the extent of the inactive semispace (for use as a marking stack,
2514 // or to zap it). Notice: space-addresses are not necessarily on the
2515 // same page, so FromSpaceStart() might be above FromSpaceEnd().
2516 Address FromSpacePageLow() { return from_space_.page_low(); }
2517 Address FromSpacePageHigh() { return from_space_.page_high(); }
2518 Address FromSpaceStart() { return from_space_.space_start(); }
2519 Address FromSpaceEnd() { return from_space_.space_end(); }
2521 // Get the extent of the active semispace's pages' memory.
2522 Address ToSpaceStart() { return to_space_.space_start(); }
2523 Address ToSpaceEnd() { return to_space_.space_end(); }
2525 inline bool ToSpaceContains(Address address) {
2526 return to_space_.Contains(address);
2528 inline bool FromSpaceContains(Address address) {
2529 return from_space_.Contains(address);
2532 // True if the object is a heap object in the address range of the
2533 // respective semispace (not necessarily below the allocation pointer of the
2535 inline bool ToSpaceContains(Object* o) { return to_space_.Contains(o); }
2536 inline bool FromSpaceContains(Object* o) { return from_space_.Contains(o); }
2538 // Try to switch the active semispace to a new, empty, page.
2539 // Returns false if this isn't possible or reasonable (i.e., there
2540 // are no pages, or the current page is already empty), or true
2542 bool AddFreshPage();
2545 // Verify the active semispace.
2546 virtual void Verify();
2550 // Print the active semispace.
2551 virtual void Print() { to_space_.Print(); }
2554 // Iterates the active semispace to collect statistics.
2555 void CollectStatistics();
2556 // Reports previously collected statistics of the active semispace.
2557 void ReportStatistics();
2558 // Clears previously collected statistics.
2559 void ClearHistograms();
2561 // Record the allocation or promotion of a heap object. Note that we don't
2562 // record every single allocation, but only those that happen in the
2563 // to space during a scavenge GC.
2564 void RecordAllocation(HeapObject* obj);
2565 void RecordPromotion(HeapObject* obj);
2567 // Return whether the operation succeded.
2568 bool CommitFromSpaceIfNeeded() {
2569 if (from_space_.is_committed()) return true;
2570 return from_space_.Commit();
2573 bool UncommitFromSpace() {
2574 if (!from_space_.is_committed()) return true;
2575 return from_space_.Uncommit();
2578 inline intptr_t inline_allocation_limit_step() {
2579 return inline_allocation_limit_step_;
2582 SemiSpace* active_space() { return &to_space_; }
2585 // Update allocation info to match the current to-space page.
2586 void UpdateAllocationInfo();
2588 Address chunk_base_;
2589 uintptr_t chunk_size_;
2592 SemiSpace to_space_;
2593 SemiSpace from_space_;
2594 base::VirtualMemory reservation_;
2597 // Start address and bit mask for containment testing.
2599 uintptr_t address_mask_;
2600 uintptr_t object_mask_;
2601 uintptr_t object_expected_;
2603 // Allocation pointer and limit for normal allocation and allocation during
2604 // mark-compact collection.
2605 AllocationInfo allocation_info_;
2607 // When incremental marking is active we will set allocation_info_.limit
2608 // to be lower than actual limit and then will gradually increase it
2609 // in steps to guarantee that we do incremental marking steps even
2610 // when all allocation is performed from inlined generated code.
2611 intptr_t inline_allocation_limit_step_;
2613 Address top_on_previous_step_;
2615 HistogramInfo* allocated_histogram_;
2616 HistogramInfo* promoted_histogram_;
2618 MUST_USE_RESULT AllocationResult SlowAllocateRaw(int size_in_bytes);
2620 friend class SemiSpaceIterator;
2623 TRACK_MEMORY("NewSpace")
2627 // -----------------------------------------------------------------------------
2628 // Old object space (excluding map objects)
2630 class OldSpace : public PagedSpace {
2632 // Creates an old space object with a given maximum capacity.
2633 // The constructor does not allocate pages from OS.
2634 OldSpace(Heap* heap, intptr_t max_capacity, AllocationSpace id,
2635 Executability executable)
2636 : PagedSpace(heap, max_capacity, id, executable) {}
2639 TRACK_MEMORY("OldSpace")
2643 // For contiguous spaces, top should be in the space (or at the end) and limit
2644 // should be the end of the space.
2645 #define DCHECK_SEMISPACE_ALLOCATION_INFO(info, space) \
2646 SLOW_DCHECK((space).page_low() <= (info).top() && \
2647 (info).top() <= (space).page_high() && \
2648 (info).limit() <= (space).page_high())
2651 // -----------------------------------------------------------------------------
2652 // Old space for all map objects
2654 class MapSpace : public PagedSpace {
2656 // Creates a map space object with a maximum capacity.
2657 MapSpace(Heap* heap, intptr_t max_capacity, AllocationSpace id)
2658 : PagedSpace(heap, max_capacity, id, NOT_EXECUTABLE),
2659 max_map_space_pages_(kMaxMapPageIndex - 1) {}
2661 // Given an index, returns the page address.
2662 // TODO(1600): this limit is artifical just to keep code compilable
2663 static const int kMaxMapPageIndex = 1 << 16;
2665 virtual int RoundSizeDownToObjectAlignment(int size) {
2666 if (base::bits::IsPowerOfTwo32(Map::kSize)) {
2667 return RoundDown(size, Map::kSize);
2669 return (size / Map::kSize) * Map::kSize;
2674 virtual void VerifyObject(HeapObject* obj);
2677 static const int kMapsPerPage = Page::kMaxRegularHeapObjectSize / Map::kSize;
2679 // Do map space compaction if there is a page gap.
2680 int CompactionThreshold() {
2681 return kMapsPerPage * (max_map_space_pages_ - 1);
2684 const int max_map_space_pages_;
2687 TRACK_MEMORY("MapSpace")
2691 // -----------------------------------------------------------------------------
2692 // Old space for simple property cell objects
2694 class CellSpace : public PagedSpace {
2696 // Creates a property cell space object with a maximum capacity.
2697 CellSpace(Heap* heap, intptr_t max_capacity, AllocationSpace id)
2698 : PagedSpace(heap, max_capacity, id, NOT_EXECUTABLE) {}
2700 virtual int RoundSizeDownToObjectAlignment(int size) {
2701 if (base::bits::IsPowerOfTwo32(Cell::kSize)) {
2702 return RoundDown(size, Cell::kSize);
2704 return (size / Cell::kSize) * Cell::kSize;
2709 virtual void VerifyObject(HeapObject* obj);
2712 TRACK_MEMORY("CellSpace")
2716 // -----------------------------------------------------------------------------
2717 // Old space for all global object property cell objects
2719 class PropertyCellSpace : public PagedSpace {
2721 // Creates a property cell space object with a maximum capacity.
2722 PropertyCellSpace(Heap* heap, intptr_t max_capacity, AllocationSpace id)
2723 : PagedSpace(heap, max_capacity, id, NOT_EXECUTABLE) {}
2725 virtual int RoundSizeDownToObjectAlignment(int size) {
2726 if (base::bits::IsPowerOfTwo32(PropertyCell::kSize)) {
2727 return RoundDown(size, PropertyCell::kSize);
2729 return (size / PropertyCell::kSize) * PropertyCell::kSize;
2734 virtual void VerifyObject(HeapObject* obj);
2737 TRACK_MEMORY("PropertyCellSpace")
2741 // -----------------------------------------------------------------------------
2742 // Large objects ( > Page::kMaxHeapObjectSize ) are allocated and managed by
2743 // the large object space. A large object is allocated from OS heap with
2744 // extra padding bytes (Page::kPageSize + Page::kObjectStartOffset).
2745 // A large object always starts at Page::kObjectStartOffset to a page.
2746 // Large objects do not move during garbage collections.
2748 class LargeObjectSpace : public Space {
2750 LargeObjectSpace(Heap* heap, intptr_t max_capacity, AllocationSpace id);
2751 virtual ~LargeObjectSpace() {}
2753 // Initializes internal data structures.
2756 // Releases internal resources, frees objects in this space.
2759 static intptr_t ObjectSizeFor(intptr_t chunk_size) {
2760 if (chunk_size <= (Page::kPageSize + Page::kObjectStartOffset)) return 0;
2761 return chunk_size - Page::kPageSize - Page::kObjectStartOffset;
2764 // Shared implementation of AllocateRaw, AllocateRawCode and
2765 // AllocateRawFixedArray.
2766 MUST_USE_RESULT AllocationResult
2767 AllocateRaw(int object_size, Executability executable);
2769 bool CanAllocateSize(int size) { return Size() + size <= max_capacity_; }
2771 // Available bytes for objects in this space.
2772 inline intptr_t Available();
2774 virtual intptr_t Size() { return size_; }
2776 virtual intptr_t SizeOfObjects() { return objects_size_; }
2778 intptr_t MaximumCommittedMemory() { return maximum_committed_; }
2780 intptr_t CommittedMemory() { return Size(); }
2782 // Approximate amount of physical memory committed for this space.
2783 size_t CommittedPhysicalMemory();
2785 int PageCount() { return page_count_; }
2787 // Finds an object for a given address, returns a Smi if it is not found.
2788 // The function iterates through all objects in this space, may be slow.
2789 Object* FindObject(Address a);
2791 // Finds a large object page containing the given address, returns NULL
2792 // if such a page doesn't exist.
2793 LargePage* FindPage(Address a);
2795 // Frees unmarked objects.
2796 void FreeUnmarkedObjects();
2798 // Checks whether a heap object is in this space; O(1).
2799 bool Contains(HeapObject* obj);
2801 // Checks whether the space is empty.
2802 bool IsEmpty() { return first_page_ == NULL; }
2804 LargePage* first_page() { return first_page_; }
2807 virtual void Verify();
2811 virtual void Print();
2812 void ReportStatistics();
2813 void CollectCodeStatistics();
2815 // Checks whether an address is in the object area in this space. It
2816 // iterates all objects in the space. May be slow.
2817 bool SlowContains(Address addr) { return FindObject(addr)->IsHeapObject(); }
2820 intptr_t max_capacity_;
2821 intptr_t maximum_committed_;
2822 // The head of the linked list of large object chunks.
2823 LargePage* first_page_;
2824 intptr_t size_; // allocated bytes
2825 int page_count_; // number of chunks
2826 intptr_t objects_size_; // size of objects
2827 // Map MemoryChunk::kAlignment-aligned chunks to large pages covering them
2830 friend class LargeObjectIterator;
2833 TRACK_MEMORY("LargeObjectSpace")
2837 class LargeObjectIterator : public ObjectIterator {
2839 explicit LargeObjectIterator(LargeObjectSpace* space);
2840 LargeObjectIterator(LargeObjectSpace* space, HeapObjectCallback size_func);
2844 // implementation of ObjectIterator.
2845 virtual HeapObject* next_object() { return Next(); }
2848 LargePage* current_;
2849 HeapObjectCallback size_func_;
2853 // Iterates over the chunks (pages and large object pages) that can contain
2854 // pointers to new space.
2855 class PointerChunkIterator BASE_EMBEDDED {
2857 inline explicit PointerChunkIterator(Heap* heap);
2859 // Return NULL when the iterator is done.
2860 MemoryChunk* next() {
2862 case kOldPointerState: {
2863 if (old_pointer_iterator_.has_next()) {
2864 return old_pointer_iterator_.next();
2870 if (map_iterator_.has_next()) {
2871 return map_iterator_.next();
2873 state_ = kLargeObjectState;
2876 case kLargeObjectState: {
2877 HeapObject* heap_object;
2879 heap_object = lo_iterator_.Next();
2880 if (heap_object == NULL) {
2881 state_ = kFinishedState;
2884 // Fixed arrays are the only pointer-containing objects in large
2886 } while (!heap_object->IsFixedArray());
2887 MemoryChunk* answer = MemoryChunk::FromAddress(heap_object->address());
2890 case kFinishedState:
2901 enum State { kOldPointerState, kMapState, kLargeObjectState, kFinishedState };
2903 PageIterator old_pointer_iterator_;
2904 PageIterator map_iterator_;
2905 LargeObjectIterator lo_iterator_;
2910 struct CommentStatistic {
2911 const char* comment;
2919 // Must be small, since an iteration is used for lookup.
2920 static const int kMaxComments = 64;
2924 } // namespace v8::internal
2926 #endif // V8_HEAP_SPACES_H_